You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by rm...@apache.org on 2018/02/22 20:14:12 UTC
[01/50] [abbrv] metron git commit: METRON-1400: Elasticsearch service
check fails in Ambari (mmiklavc via mmiklavc) closes apache/metron#904
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1344-test-infrastructure ebc325865 -> fc4ce0104
METRON-1400: Elasticsearch service check fails in Ambari (mmiklavc via mmiklavc) closes apache/metron#904
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9d582eac
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9d582eac
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9d582eac
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 9d582eacd670908ef076da0cef4bd9f566c892c8
Parents: 173ce50
Author: mmiklavc <mi...@gmail.com>
Authored: Wed Jan 24 15:30:00 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Wed Jan 24 15:30:00 2018 -0700
----------------------------------------------------------------------
.../ELASTICSEARCH/5.6.2/package/scripts/service_check.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/9d582eac/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
index d59954f..4f8b1ab 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
@@ -19,6 +19,7 @@ limitations under the License.
"""
import subprocess
import sys
+import re
from resource_management.core.resources.system import Execute
from resource_management.libraries.script import Script
@@ -32,12 +33,14 @@ class ServiceCheck(Script):
doc = '{"name": "Ambari Smoke test"}'
index = "ambari_smoke_test"
- Logger.info("Running Elastic search service check", file=sys.stdout)
+ # http_port from ES config may be a port range.
+ es_http_port = re.search("^(\d+)", params.http_port).group(1)
+ host = params.hostname + ":" + es_http_port
+ Logger.info("Running Elastic search service check against " + host)
# Make sure the service is actually up. We can live without everything allocated.
# Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can
# also hit ES before things are green.
- host = "localhost:9200"
Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host,
logoutput=True,
tries=6,
[11/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/files/yaf.out
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/files/yaf.out b/metron-deployment/roles/sensor-stubs/files/yaf.out
deleted file mode 100644
index b7c4710..0000000
--- a/metron-deployment/roles/sensor-stubs/files/yaf.out
+++ /dev/null
@@ -1,22164 +0,0 @@
-2017-01-11 20:52:24.012|2017-01-11 20:52:24.012| 0.000| 0.000| 6| 192.168.138.158|49195| 188.165.164.184| 80| AR| 0| 0| 0|522c999d|00000000|000|000| 1| 40| 0| 0| 0|
-2017-01-11 20:52:24.008|2017-01-11 20:52:24.022| 0.014| 0.002| 6| 72.34.49.86| 80| 192.168.138.158|49204| AP| APF| AF| A|3e05e129|5869e531|000|000| 3| 358| 2| 80| 0|
-2017-01-11 20:52:36.640|2017-01-11 20:53:28.138| 51.498| 0.000| 6| 62.75.195.236| 80| 192.168.138.158|49186| AP| AP| AR| A|73734c1c|516c4a0c|000|000| 2| 82| 3| 120| 0|
-2017-01-11 20:52:52.845|2017-01-11 20:53:58.724| 65.879| 0.187| 6| 192.168.138.158|49208| 95.163.121.204| 80| S| APR| AS| AP|be149584|8ef40c65|000|000| 7| 1206| 6| 2873| 0|
-2017-01-11 20:52:52.845|2017-01-11 20:54:03.711| 70.866| 0.187| 6| 192.168.138.158|49207| 95.163.121.204| 80| S| APR| AS| AP|c031307b|d1fe0059|000|000| 7| 1088| 5| 1685| 0|
-2017-01-11 20:52:52.845|2017-01-11 20:54:08.699| 75.854| 0.201| 6| 192.168.138.158|49209| 95.163.121.204| 80| S| APR| AS| AP|32c0b55e|f31124b8|000|000| 8| 1502| 9| 6752| 0|
-2017-01-11 20:52:50.865|2017-01-11 20:54:08.699| 77.834| 0.173| 6| 192.168.138.158|49206| 95.163.121.204| 80| S| APR| AS| AP|a80daa6e|b93a04df|000|000| 10| 2265| 12| 7194| 0|
-2017-01-11 20:52:48.208|2017-01-11 20:54:08.700| 80.492| 0.172| 6| 192.168.138.158|49205| 95.163.121.204| 80| S| APR| AS| AP|ca8d81e2|0628e3a4|000|000| 11| 2097| 14| 9627| 0|
-2017-01-11 20:52:52.847|2017-01-11 20:54:08.700| 75.853| 0.185| 6| 192.168.138.158|49210| 95.163.121.204| 80| S| APR| AS| AP|63626a70|9b7a5687|000|000| 8| 1645| 7| 2238| 0|
-2017-01-11 20:54:08.935|2017-01-11 20:54:09.448| 0.513| 0.132| 6| 192.168.138.158|49184| 62.75.195.236| 80| S| APF| AS| APF|92a7a033|00b98442|000|000| 5| 805| 5| 966| 0|
-2017-01-11 20:54:09.637|2017-01-11 20:54:10.054| 0.417| 0.131| 6| 192.168.138.158|49185| 62.75.195.236| 80| S| APF| AS| APF|14dad2de|057d07a2|000|000| 6| 782| 10| 9600| 0|
-2017-01-11 20:54:10.747|2017-01-11 20:54:11.162| 0.415| 0.136| 6| 192.168.138.158|49188| 62.75.195.236| 80| S| APF| AS| APF|f017c491|abdb7ffd|000|000| 5| 283| 5| 1267| 0|
-2017-01-11 20:54:11.153|2017-01-11 20:54:12.135| 0.982| 0.135| 6| 192.168.138.158|49189| 62.75.195.236| 80| S| APF| AS| APF|9dfb17ed|f1bbe6ec|000|000| 80| 3525| 167| 229440| 0|
-2017-01-11 20:54:13.341|2017-01-11 20:54:13.628| 0.287| 0.131| 6| 192.168.138.158|49190| 62.75.195.236| 80| S| APF| AS| APF|abe3db05|2178457a|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:54:13.630|2017-01-11 20:54:13.941| 0.311| 0.153| 6| 192.168.138.158|49191| 62.75.195.236| 80| S| APF| AS| APF|6ac7f41d|1ecaba70|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:54:13.943|2017-01-11 20:54:14.480| 0.537| 0.155| 6| 192.168.138.158|49192| 62.75.195.236| 80| S| APF| AS| APF|d57e1fff|00b640f3|000|000| 5| 525| 5| 418| 0|
-2017-01-11 20:54:14.483|2017-01-11 20:54:14.793| 0.310| 0.149| 6| 192.168.138.158|49193| 62.75.195.236| 80| S| APF| AS| APF|1cff6791|0099d801|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:54:14.795|2017-01-11 20:54:15.093| 0.298| 0.140| 6| 192.168.138.158|49194| 62.75.195.236| 80| S| APF| AS| APF|d7003ea2|c8807b47|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:54:15.133|2017-01-11 20:54:15.334| 0.201| 0.042| 6| 192.168.138.158|49197| 204.152.254.221| 80| S| APF| AS| APF|58cf2c67|166f1d89|000|000| 7| 820| 6| 765| 0|
-2017-01-11 20:54:15.094|2017-01-11 20:54:15.388| 0.294| 0.138| 6| 192.168.138.158|49196| 62.75.195.236| 80| S| APF| AS| APF|bdd9bac5|546ac731|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:54:15.555|2017-01-11 20:54:17.711| 2.156| 0.051| 6| 192.168.138.158|49198| 72.34.49.86| 80| S| APF| AS| APF|24718a4a|e263a145|000|000| 7| 822| 7| 522| 0|
-2017-01-11 20:54:17.700|2017-01-11 20:54:17.886| 0.186| 0.059| 6| 192.168.138.158|49199| 204.152.254.221| 80| S| APF| AS| APF|86fa40fb|20de0881|000|000| 6| 741| 5| 725| 0|
-2017-01-11 20:54:17.889|2017-01-11 20:54:20.677| 2.788| 0.103| 6| 192.168.138.158|49200| 72.34.49.86| 80| S| APF| AS| APF|788e6fcc|271a3fd2|000|000| 6| 743| 6| 1466| 0|
-2017-01-11 20:54:20.666|2017-01-11 20:54:20.832| 0.166| 0.053| 6| 192.168.138.158|49201| 204.152.254.221| 80| S| APF| AS| APF|b74faf85|50818fd9|000|000| 7| 846| 6| 765| 0|
-2017-01-11 20:54:20.834|2017-01-11 20:54:24.431| 3.597| 0.106| 6| 192.168.138.158|49202| 72.34.49.86| 80| S| APF| AS| APF|78494ef3|0d188aea|000|000| 19| 1328| 39| 47453| 0|
-2017-01-11 20:54:27.057|2017-01-11 20:54:27.252| 0.195| 0.043| 6| 192.168.138.158|49203| 204.152.254.221| 80| S| APF| AS| APF|075be34c|00357320|000|000| 6| 754| 5| 725| 0|
-2017-01-11 20:54:14.819|2017-01-11 20:54:29.238| 14.419| 0.129| 6| 192.168.138.158|49195| 188.165.164.184| 80| S| APR| AS| AP|522c98b3|c88832bc|000|000| 5| 445| 3| 552| 0|
-2017-01-11 20:54:27.255|2017-01-11 20:54:29.248| 1.993| 0.105| 6| 192.168.138.158|49204| 72.34.49.86| 80| S| APF| AS| APF|5869e338|3e05e128|000|000| 6| 756| 6| 482| 0|
-2017-01-11 20:54:09.651|2017-01-11 20:55:33.365| 83.714| 0.131| 6| 192.168.138.158|49186| 62.75.195.236| 80| S| APR| AS| AP|516c475c|7371702c|000|000| 51| 2739| 100| 125851| 0|
-2017-01-11 20:54:58.071|2017-01-11 20:56:03.951| 65.880| 0.187| 6| 192.168.138.158|49208| 95.163.121.204| 80| S| APR| AS| AP|be149584|8ef40c65|000|000| 7| 1206| 6| 2873| 0|
-2017-01-11 20:54:58.070|2017-01-11 20:56:08.937| 70.867| 0.188| 6| 192.168.138.158|49207| 95.163.121.204| 80| S| APR| AS| AP|c031307b|d1fe0059|000|000| 7| 1088| 5| 1685| 0|
-2017-01-11 20:54:58.071|2017-01-11 20:56:13.926| 75.855| 0.201| 6| 192.168.138.158|49209| 95.163.121.204| 80| S| APR| AS| AP|32c0b55e|f31124b8|000|000| 8| 1502| 9| 6752| 0|
-2017-01-11 20:54:56.089|2017-01-11 20:56:13.926| 77.837| 0.174| 6| 192.168.138.158|49206| 95.163.121.204| 80| S| APR| AS| AP|a80daa6e|b93a04df|000|000| 10| 2265| 12| 7194| 0|
-2017-01-11 20:54:53.434|2017-01-11 20:56:13.926| 80.492| 0.173| 6| 192.168.138.158|49205| 95.163.121.204| 80| S| APR| AS| AP|ca8d81e2|0628e3a4|000|000| 11| 2097| 14| 9627| 0|
-2017-01-11 20:54:58.073|2017-01-11 20:56:13.926| 75.853| 0.185| 6| 192.168.138.158|49210| 95.163.121.204| 80| S| APR| AS| AP|63626a70|9b7a5687|000|000| 8| 1645| 7| 2238| 0|
-2017-01-11 20:56:14.161|2017-01-11 20:56:14.672| 0.511| 0.133| 6| 192.168.138.158|49184| 62.75.195.236| 80| S| APF| AS| APF|92a7a033|00b98442|000|000| 5| 805| 5| 966| 0|
-2017-01-11 20:56:14.861|2017-01-11 20:56:15.278| 0.417| 0.132| 6| 192.168.138.158|49185| 62.75.195.236| 80| S| APF| AS| APF|14dad2de|057d07a2|000|000| 6| 782| 10| 9600| 0|
-2017-01-11 20:56:15.960|2017-01-11 20:56:16.373| 0.413| 0.136| 6| 192.168.138.158|49188| 62.75.195.236| 80| S| APF| AS| APF|f017c491|abdb7ffd|000|000| 5| 283| 5| 1267| 0|
-2017-01-11 20:56:16.364|2017-01-11 20:56:17.370| 1.006| 0.135| 6| 192.168.138.158|49189| 62.75.195.236| 80| S| APF| AS| APF|9dfb17ed|f1bbe6ec|000|000| 80| 3525| 167| 229440| 0|
-2017-01-11 20:56:18.575|2017-01-11 20:56:18.864| 0.289| 0.131| 6| 192.168.138.158|49190| 62.75.195.236| 80| S| APF| AS| APF|abe3db05|2178457a|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:56:18.865|2017-01-11 20:56:19.176| 0.311| 0.153| 6| 192.168.138.158|49191| 62.75.195.236| 80| S| APF| AS| APF|6ac7f41d|1ecaba70|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:56:19.177|2017-01-11 20:56:19.712| 0.535| 0.154| 6| 192.168.138.158|49192| 62.75.195.236| 80| S| APF| AS| APF|d57e1fff|00b640f3|000|000| 5| 525| 5| 418| 0|
-2017-01-11 20:56:19.715|2017-01-11 20:56:20.025| 0.310| 0.150| 6| 192.168.138.158|49193| 62.75.195.236| 80| S| APF| AS| APF|1cff6791|0099d801|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:56:20.027|2017-01-11 20:56:20.325| 0.298| 0.140| 6| 192.168.138.158|49194| 62.75.195.236| 80| S| APF| AS| APF|d7003ea2|c8807b47|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:56:20.365|2017-01-11 20:56:20.567| 0.202| 0.042| 6| 192.168.138.158|49197| 204.152.254.221| 80| S| APF| AS| APF|58cf2c67|166f1d89|000|000| 7| 820| 6| 765| 0|
-2017-01-11 20:56:20.326|2017-01-11 20:56:20.620| 0.294| 0.139| 6| 192.168.138.158|49196| 62.75.195.236| 80| S| APF| AS| APF|bdd9bac5|546ac731|000|000| 5| 525| 4| 378| 0|
-2017-01-11 20:56:20.787|2017-01-11 20:56:22.941| 2.154| 0.051| 6| 192.168.138.158|49198| 72.34.49.86| 80| S| APF| AS| APF|24718a4a|e263a145|000|000| 7| 822| 7| 522| 0|
-2017-01-11 20:56:22.930|2017-01-11 20:56:23.114| 0.184| 0.059| 6| 192.168.138.158|49199| 204.152.254.221| 80| S| APF| AS| APF|86fa40fb|20de0881|000|000| 6| 741| 5| 725| 0|
-2017-01-11 20:56:23.118|2017-01-11 20:56:25.907| 2.789| 0.105| 6| 192.168.138.158|49200| 72.34.49.86| 80| S| APF| AS| APF|788e6fcc|271a3fd2|000|000| 6| 743| 6| 1466| 0|
-2017-01-11 20:56:25.896|2017-01-11 20:56:26.062| 0.166| 0.054| 6| 192.168.138.158|49201| 204.152.254.221| 80| S| APF| AS| APF|b74faf85|50818fd9|000|000| 7| 846| 6| 765| 0|
-2017-01-11 20:56:26.064|2017-01-11 20:56:29.665| 3.601| 0.106| 6| 192.168.138.158|49202| 72.34.49.86| 80| S| APF| AS| APF|78494ef3|0d188aea|000|000| 19| 1328| 39| 47453| 0|
-2017-01-11 20:56:32.291|2017-01-11 20:56:32.484| 0.193| 0.042| 6| 192.168.138.158|49203| 204.152.254.221| 80| S| APF| AS| APF|075be34c|00357320|000|000| 6| 754| 5| 725| 0|
-2017-01-11 20:56:20.052|2017-01-11 20:56:34.469| 14.417| 0.128| 6| 192.168.138.158|49195| 188.165.164.184| 80| S| APR| AS| AP|522c98b3|c88832bc|000|000| 5| 445| 3| 552| 0|
-2017-01-11 20:56:32.486|2017-01-11 20:56:34.479| 1.993| 0.105| 6| 192.168.138.158|49204| 72.34.49.86| 80| S| APF| AS| APF|5869e338|3e05e128|000|000| 6| 756| 6| 482| 0|
-2017-01-11 20:53:15.474|2017-01-11 20:56:10.859| 175.385| 0.000| 17| 192.168.66.1| 5353| 224.0.0.251| 5353| 0| 0| 0| 0|00000000|00000000|000|000| 12| 816| 0| 0| 0|eof
-2017-01-11 20:54:08.700|2017-01-11 20:56:14.077| 125.377| 0.151| 17| 192.168.138.158|60078| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 246| 2| 278| 0|eof
-2017-01-11 20:54:09.492|2017-01-11 20:56:14.861| 125.369| 0.144| 17| 192.168.138.158|65315| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 244| 2| 276| 0|eof
-2017-01-11 20:54:09.492|2017-01-11 20:56:14.875| 125.383| 0.158| 17| 192.168.138.158|50683| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 242| 2| 274| 0|eof
-2017-01-11 20:52:23.006|2017-01-11 20:56:17.746| 234.740| 0.022| 6| 192.168.66.1|50181| 192.168.66.121| 8080| AP| AP| AP| AP|89de8804|413104f3|000|000| 20| 6226| 14| 6653| 0|eof
-2017-01-11 20:54:14.677|2017-01-11 20:56:20.051| 125.374| 0.142| 17| 192.168.138.158|53571| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 112| 2| 144| 0|eof
-2017-01-11 20:54:15.095|2017-01-11 20:56:20.364| 125.269| 0.037| 17| 192.168.138.158|61720| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 112| 2| 144| 0|eof
-2017-01-11 20:54:15.336|2017-01-11 20:56:20.753| 125.417| 0.185| 17| 192.168.138.158|50509| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 150| 2| 262| 0|eof
-2017-01-11 20:54:15.522|2017-01-11 20:56:20.786| 125.264| 0.033| 17| 192.168.138.158|56753| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 128| 2| 160| 0|eof
-2017-01-11 20:52:23.007|2017-01-11 20:56:24.203| 241.196| 0.039| 6| 192.168.66.1|50186| 192.168.66.121| 8080| AP| AP| AP| AP|e6b20f8c|79c1af21|000|000| 75| 34066| 56| 30255| 0|eof
-2017-01-11 20:56:14.875|2017-01-11 20:56:47.098| 32.223| 0.131| 6| 192.168.138.158|49186| 62.75.195.236| 80| S| AP| AS| AP|516c475c|7371702c|000|000| 49| 2659| 99| 125810| 0|eof
-2017-01-11 20:52:23.007|2017-01-11 20:56:52.666| 269.659| 0.011| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| AP| AP| AP|7b810bc7|f502ff81|000|000| 76| 28692| 56| 30587| 0|eof
-2017-01-11 20:52:23.005|2017-01-11 20:56:52.686| 269.681| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| AP| AP| AP|e53b601f|185bd4f0|000|000| 96| 38685| 70| 39312| 0|eof
-2017-01-11 20:52:23.007|2017-01-11 20:56:53.683| 270.676| 0.040| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| AP| AP| AP|3982f0c2|a885b507|000|000| 235| 95979| 174| 101932| 0|eof
-2017-01-11 20:52:23.006|2017-01-11 20:56:56.041| 273.035| 0.008| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| AP| AP| AP|83665d75|8df32014|000|000| 332| 146689| 257| 146440| 0|eof
-2017-01-11 20:52:48.175|2017-01-11 20:56:58.664| 250.489| 0.032| 17| 192.168.138.158|50329| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 3| 231| 3| 279| 0|eof
-2017-01-11 20:56:58.838|2017-01-11 20:56:58.838| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AS| 0| 0| 0|0628e3a4|00000000|000|000| 1| 44| 0| 0| 0|idle
-2017-01-11 20:56:58.839|2017-01-11 20:56:58.839| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d81e3|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:56:58.840|2017-01-11 20:56:58.840| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d81e3|00000000|000|000| 1| 369| 0| 0| 0|idle
-2017-01-11 20:56:58.840|2017-01-11 20:56:58.840| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628e3a5|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:56:59.992|2017-01-11 20:56:59.992| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83685706|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:56:59.992|2017-01-11 20:56:59.992| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83685cae|00000000|000|000| 1| 908| 0| 0| 0|idle
-2017-01-11 20:56:59.992|2017-01-11 20:56:59.992| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df527e8|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|398437fd|00000000|000|000| 1| 864| 0| 0| 0|idle
-2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53be3e0|00000000|000|000| 1| 580| 0| 0| 0|idle
-2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6048|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b816c8f|00000000|000|000| 1| 629| 0| 0| 0|idle
-2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| A| 0| 0| 0|f5036b9c|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:56:59.995|2017-01-11 20:56:59.995| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6048|00000000|000|000| 1| 302| 0| 0| 0|idle
-2017-01-11 20:56:59.995|2017-01-11 20:56:59.995| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53be5f0|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:00.010|2017-01-11 20:57:00.010| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f5036b9c|00000000|000|000| 1| 1032| 0| 0| 0|idle
-2017-01-11 20:57:00.010|2017-01-11 20:57:00.010| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b816ed0|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:00.032|2017-01-11 20:57:00.032| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8871fdb|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:00.069|2017-01-11 20:57:00.069| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df527e8|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:00.069|2017-01-11 20:57:00.069| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df52d90|00000000|000|000| 1| 189| 0| 0| 0|idle
-2017-01-11 20:57:00.069|2017-01-11 20:57:00.069| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686006|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:00.095|2017-01-11 20:57:00.095| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8871fdb|00000000|000|000| 1| 1067| 0| 0| 0|idle
-2017-01-11 20:57:00.095|2017-01-11 20:57:00.095| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843b29|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.317|2017-01-11 20:57:01.317| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628e3a5|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:01.322|2017-01-11 20:57:01.322| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| S| 0| 0| 0|a80daa6e|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.322|2017-01-11 20:57:01.322| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628e8f0|00000000|000|000| 1| 393| 0| 0| 0|idle
-2017-01-11 20:57:01.323|2017-01-11 20:57:01.323| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d832c|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:01.495|2017-01-11 20:57:01.495| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AS| 0| 0| 0|b93a04df|00000000|000|000| 1| 44| 0| 0| 0|idle
-2017-01-11 20:57:01.495|2017-01-11 20:57:01.495| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80daa6f|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:01.496|2017-01-11 20:57:01.496| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80daa6f|00000000|000|000| 1| 478| 0| 0| 0|idle
-2017-01-11 20:57:01.496|2017-01-11 20:57:01.496| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a04e0|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:01.942|2017-01-11 20:57:01.942| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|39843b29|00000000|000|000| 1| 504| 0| 0| 0|idle
-2017-01-11 20:57:01.942|2017-01-11 20:57:01.942| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a88723d2|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.943|2017-01-11 20:57:01.943| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686006|00000000|000|000| 1| 628| 0| 0| 0|idle
-2017-01-11 20:57:01.943|2017-01-11 20:57:01.943| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b816ed0|00000000|000|000| 1| 678| 0| 0| 0|idle
-2017-01-11 20:57:01.948|2017-01-11 20:57:01.948| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df52e19|00000000|000|000| 1| 502| 0| 0| 0|idle
-2017-01-11 20:57:01.948|2017-01-11 20:57:01.948| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686246|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.950|2017-01-11 20:57:01.950| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f5036f70|00000000|000|000| 1| 544| 0| 0| 0|idle
-2017-01-11 20:57:01.950|2017-01-11 20:57:01.950| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817142|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a88723d2|00000000|000|000| 1| 2948| 0| 0| 0|idle
-2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8872f22|00000000|000|000| 1| 151| 0| 0| 0|idle
-2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843ced|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843ced|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:01.990|2017-01-11 20:57:01.990| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843ced|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:03.212|2017-01-11 20:57:03.212| 0.000| 0.000| 6| 192.168.66.1|50181| 192.168.66.121| 8080| A| 0| 0| 0|89de9c75|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.212|2017-01-11 20:57:03.212| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50181| A| 0| 0| 0|41311c18|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:03.281|2017-01-11 20:57:03.281| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a04e0|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:03.295|2017-01-11 20:57:03.295| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a0a2b|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:03.295|2017-01-11 20:57:03.295| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a0f76|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:03.295|2017-01-11 20:57:03.295| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a152a|00000000|000|000| 1| 636| 0| 0| 0|idle
-2017-01-11 20:57:03.296|2017-01-11 20:57:03.296| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80dac25|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d832c|00000000|000|000| 1| 481| 0| 0| 0|idle
-2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628ea51|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80dac25|00000000|000|000| 1| 481| 0| 0| 0|idle
-2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a177e|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.302|2017-01-11 20:57:03.302| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| S| 0| 0| 0|c031307b|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:03.302|2017-01-11 20:57:03.302| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| S| 0| 0| 0|be149584|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:03.302|2017-01-11 20:57:03.302| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| S| 0| 0| 0|32c0b55e|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:03.304|2017-01-11 20:57:03.304| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| S| 0| 0| 0|63626a70|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:03.489|2017-01-11 20:57:03.489| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AS| 0| 0| 0|8ef40c65|00000000|000|000| 1| 44| 0| 0| 0|idle
-2017-01-11 20:57:03.489|2017-01-11 20:57:03.489| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| AS| 0| 0| 0|d1fe0059|00000000|000|000| 1| 44| 0| 0| 0|idle
-2017-01-11 20:57:03.489|2017-01-11 20:57:03.489| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AS| 0| 0| 0|9b7a5687|00000000|000|000| 1| 44| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| A| 0| 0| 0|be149585|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| A| 0| 0| 0|c031307c|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| A| 0| 0| 0|63626a71|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| AP| 0| 0| 0|be149585|00000000|000|000| 1| 519| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| A| 0| 0| 0|8ef40c66|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| AP| 0| 0| 0|c031307c|00000000|000|000| 1| 481| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| A| 0| 0| 0|d1fe005a|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| AP| 0| 0| 0|63626a71|00000000|000|000| 1| 475| 0| 0| 0|idle
-2017-01-11 20:57:03.491|2017-01-11 20:57:03.491| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| A| 0| 0| 0|9b7a5688|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.503|2017-01-11 20:57:03.503| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AS| 0| 0| 0|f31124b8|00000000|000|000| 1| 44| 0| 0| 0|idle
-2017-01-11 20:57:03.503|2017-01-11 20:57:03.503| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b55f|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:03.506|2017-01-11 20:57:03.506| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| AP| 0| 0| 0|32c0b55f|00000000|000|000| 1| 481| 0| 0| 0|idle
-2017-01-11 20:57:03.506|2017-01-11 20:57:03.506| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| A| 0| 0| 0|f31124b9|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.068|2017-01-11 20:57:05.068| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a177e|00000000|000|000| 1| 856| 0| 0| 0|idle
-2017-01-11 20:57:05.070|2017-01-11 20:57:05.070| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80dadde|00000000|000|000| 1| 481| 0| 0| 0|idle
-2017-01-11 20:57:05.071|2017-01-11 20:57:05.071| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a1aae|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.283|2017-01-11 20:57:05.283| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628ea51|00000000|000|000| 1| 1129| 0| 0| 0|idle
-2017-01-11 20:57:05.286|2017-01-11 20:57:05.286| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d84e5|00000000|000|000| 1| 475| 0| 0| 0|idle
-2017-01-11 20:57:05.286|2017-01-11 20:57:05.286| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628ee92|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.319|2017-01-11 20:57:05.319| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AP| 0| 0| 0|9b7a5688|00000000|000|000| 1| 529| 0| 0| 0|idle
-2017-01-11 20:57:05.321|2017-01-11 20:57:05.321| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| AP| 0| 0| 0|63626c24|00000000|000|000| 1| 475| 0| 0| 0|idle
-2017-01-11 20:57:05.322|2017-01-11 20:57:05.322| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| A| 0| 0| 0|9b7a5871|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.403|2017-01-11 20:57:05.403| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AP| 0| 0| 0|8ef40c66|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:05.418|2017-01-11 20:57:05.418| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AP| 0| 0| 0|8ef411b1|00000000|000|000| 1| 828| 0| 0| 0|idle
-2017-01-11 20:57:05.418|2017-01-11 20:57:05.418| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| A| 0| 0| 0|be149764|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.419|2017-01-11 20:57:05.419| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| AP| 0| 0| 0|be149764|00000000|000|000| 1| 475| 0| 0| 0|idle
-2017-01-11 20:57:05.419|2017-01-11 20:57:05.419| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| A| 0| 0| 0|8ef414c5|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.557|2017-01-11 20:57:05.557| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| AP| 0| 0| 0|d1fe005a|00000000|000|000| 1| 938| 0| 0| 0|idle
-2017-01-11 20:57:05.658|2017-01-11 20:57:05.658| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| A| 0| 0| 0|c0313235|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:05.682|2017-01-11 20:57:05.682| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f31124b9|00000000|000|000| 1| 838| 0| 0| 0|idle
-2017-01-11 20:57:05.785|2017-01-11 20:57:05.785| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b718|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:07.100|2017-01-11 20:57:07.100| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|39843ced|00000000|000|000| 1| 580| 0| 0| 0|idle
-2017-01-11 20:57:07.100|2017-01-11 20:57:07.100| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b817142|00000000|000|000| 1| 629| 0| 0| 0|idle
-2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686246|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836867ee|00000000|000|000| 1| 908| 0| 0| 0|idle
-2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53be5f0|00000000|000|000| 1| 864| 0| 0| 0|idle
-2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8872f85|00000000|000|000| 1| 302| 0| 0| 0|idle
-2017-01-11 20:57:07.102|2017-01-11 20:57:07.102| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843efd|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:07.102|2017-01-11 20:57:07.102| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df52fdb|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:07.111|2017-01-11 20:57:07.111| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f503715c|00000000|000|000| 1| 1033| 0| 0| 0|idle
-2017-01-11 20:57:07.111|2017-01-11 20:57:07.111| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817383|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:07.142|2017-01-11 20:57:07.142| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6142|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:07.173|2017-01-11 20:57:07.173| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a1aae|00000000|000|000| 1| 998| 0| 0| 0|idle
-2017-01-11 20:57:07.219|2017-01-11 20:57:07.219| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df52fdb|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:07.219|2017-01-11 20:57:07.219| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df53583|00000000|000|000| 1| 189| 0| 0| 0|idle
-2017-01-11 20:57:07.219|2017-01-11 20:57:07.219| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686b46|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:07.234|2017-01-11 20:57:07.234| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6142|00000000|000|000| 1| 1067| 0| 0| 0|idle
-2017-01-11 20:57:07.234|2017-01-11 20:57:07.234| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53be91c|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:07.277|2017-01-11 20:57:07.277| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80daf97|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:07.345|2017-01-11 20:57:07.345| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628ee92|00000000|000|000| 1| 531| 0| 0| 0|idle
-2017-01-11 20:57:07.416|2017-01-11 20:57:07.416| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AP| 0| 0| 0|9b7a5871|00000000|000|000| 1| 528| 0| 0| 0|idle
-2017-01-11 20:57:07.445|2017-01-11 20:57:07.445| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d8698|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:07.519|2017-01-11 20:57:07.519| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| A| 0| 0| 0|63626dd7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:07.538|2017-01-11 20:57:07.538| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AP| 0| 0| 0|8ef414c5|00000000|000|000| 1| 526| 0| 0| 0|idle
-2017-01-11 20:57:07.551|2017-01-11 20:57:07.551| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| AP| 0| 0| 0|c0313235|00000000|000|000| 1| 395| 0| 0| 0|idle
-2017-01-11 20:57:07.551|2017-01-11 20:57:07.551| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| A| 0| 0| 0|d1fe03dc|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:07.639|2017-01-11 20:57:07.639| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| A| 0| 0| 0|be149917|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:09.043|2017-01-11 20:57:09.043| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53be91c|00000000|000|000| 1| 836| 0| 0| 0|idle
-2017-01-11 20:57:09.043|2017-01-11 20:57:09.043| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6539|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:09.043|2017-01-11 20:57:09.043| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686b46|00000000|000|000| 1| 608| 0| 0| 0|idle
-2017-01-11 20:57:09.080|2017-01-11 20:57:09.080| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6539|00000000|000|000| 1| 317| 0| 0| 0|idle
-2017-01-11 20:57:09.081|2017-01-11 20:57:09.081| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bec2c|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:09.083|2017-01-11 20:57:09.083| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df5360c|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:09.085|2017-01-11 20:57:09.085| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df5360c|00000000|000|000| 1| 671| 0| 0| 0|idle
-2017-01-11 20:57:09.085|2017-01-11 20:57:09.085| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686d72|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:09.525|2017-01-11 20:57:09.525| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| AP| 0| 0| 0|d1fe03dc|00000000|000|000| 1| 623| 0| 0| 0|idle
-2017-01-11 20:57:09.628|2017-01-11 20:57:09.628| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| A| 0| 0| 0|c0313398|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:09.697|2017-01-11 20:57:09.697| 0.000| 0.000| 6| 192.168.66.1|50186| 192.168.66.121| 8080| A| 0| 0| 0|e6b28585|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:09.697|2017-01-11 20:57:09.697| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50186| A| 0| 0| 0|79c219f0|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:12.491|2017-01-11 20:57:12.491| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| AP| 0| 0| 0|32c0b718|00000000|000|000| 1| 769| 0| 0| 0|idle
-2017-01-11 20:57:12.492|2017-01-11 20:57:12.492| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| A| 0| 0| 0|f31127d7|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:12.639|2017-01-11 20:57:12.639| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686d72|00000000|000|000| 1| 628| 0| 0| 0|idle
-2017-01-11 20:57:12.639|2017-01-11 20:57:12.639| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df53877|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:12.640|2017-01-11 20:57:12.640| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bec2c|00000000|000|000| 1| 678| 0| 0| 0|idle
-2017-01-11 20:57:12.644|2017-01-11 20:57:12.644| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df53877|00000000|000|000| 1| 501| 0| 0| 0|idle
-2017-01-11 20:57:12.644|2017-01-11 20:57:12.644| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686fb2|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:12.650|2017-01-11 20:57:12.650| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6642|00000000|000|000| 1| 544| 0| 0| 0|idle
-2017-01-11 20:57:12.650|2017-01-11 20:57:12.650| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bee9e|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bee9e|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf446|00000000|000|000| 1| 136| 0| 0| 0|idle
-2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c682e|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686fb2|00000000|000|000| 1| 504| 0| 0| 0|idle
-2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b817383|00000000|000|000| 1| 580| 0| 0| 0|idle
-2017-01-11 20:57:13.639|2017-01-11 20:57:13.639| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|39843efd|00000000|000|000| 1| 629| 0| 0| 0|idle
-2017-01-11 20:57:13.640|2017-01-11 20:57:13.640| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f5037531|00000000|000|000| 1| 302| 0| 0| 0|idle
-2017-01-11 20:57:13.640|2017-01-11 20:57:13.640| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817593|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.642|2017-01-11 20:57:13.642| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c682e|00000000|000|000| 1| 678| 0| 0| 0|idle
-2017-01-11 20:57:13.642|2017-01-11 20:57:13.642| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bf49a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.646|2017-01-11 20:57:13.646| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a887307f|00000000|000|000| 1| 1032| 0| 0| 0|idle
-2017-01-11 20:57:13.647|2017-01-11 20:57:13.647| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|3984413e|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.657|2017-01-11 20:57:13.657| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df53a38|00000000|000|000| 1| 2948| 0| 0| 0|idle
-2017-01-11 20:57:13.657|2017-01-11 20:57:13.657| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.657|2017-01-11 20:57:13.657| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df54588|00000000|000|000| 1| 150| 0| 0| 0|idle
-2017-01-11 20:57:13.658|2017-01-11 20:57:13.658| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:13.686|2017-01-11 20:57:13.686| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:14.652|2017-01-11 20:57:14.652| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:14.652|2017-01-11 20:57:14.652| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|8368771e|00000000|000|000| 1| 908| 0| 0| 0|idle
-2017-01-11 20:57:14.652|2017-01-11 20:57:14.652| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df545ea|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:14.653|2017-01-11 20:57:14.653| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|3984413e|00000000|000|000| 1| 864| 0| 0| 0|idle
-2017-01-11 20:57:14.693|2017-01-11 20:57:14.693| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873453|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:14.719|2017-01-11 20:57:14.719| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df545ea|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:14.719|2017-01-11 20:57:14.719| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df54b92|00000000|000|000| 1| 187| 0| 0| 0|idle
-2017-01-11 20:57:14.719|2017-01-11 20:57:14.719| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687a76|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:14.743|2017-01-11 20:57:14.743| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8873453|00000000|000|000| 1| 1067| 0| 0| 0|idle
-2017-01-11 20:57:14.743|2017-01-11 20:57:14.743| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|3984446a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:15.201|2017-01-11 20:57:15.201| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f31127d7|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f3112d22|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b9f1|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| A| 0| 0| 0|f311326d|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f3113821|00000000|000|000| 1| 1290| 0| 0| 0|idle
-2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f3113d03|00000000|000|000| 1| 210| 0| 0| 0|idle
-2017-01-11 20:57:15.216|2017-01-11 20:57:15.216| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b9f1|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:15.228|2017-01-11 20:57:15.228| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80daf97|00000000|000|000| 1| 573| 0| 0| 0|idle
-2017-01-11 20:57:15.228|2017-01-11 20:57:15.228| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a1e6c|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:17.326|2017-01-11 20:57:17.326| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a1e6c|00000000|000|000| 1| 210| 0| 0| 0|idle
-2017-01-11 20:57:17.334|2017-01-11 20:57:17.334| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d8698|00000000|000|000| 1| 480| 0| 0| 0|idle
-2017-01-11 20:57:17.335|2017-01-11 20:57:17.335| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628f07d|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:17.335|2017-01-11 20:57:17.335| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| AP| 0| 0| 0|63626dd7|00000000|000|000| 1| 483| 0| 0| 0|idle
-2017-01-11 20:57:17.335|2017-01-11 20:57:17.335| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| A| 0| 0| 0|9b7a5a59|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:17.429|2017-01-11 20:57:17.429| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80db1ac|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:18.962|2017-01-11 20:57:18.962| 0.000| 0.000| 6| 62.75.195.236| 80| 192.168.138.158|49186| AP| 0| 0| 0|73734c1c|00000000|000|000| 1| 41| 0| 0| 0|idle
-2017-01-11 20:57:18.963|2017-01-11 20:57:18.963| 0.000| 0.000| 6| 192.168.138.158|49186| 62.75.195.236| 80| A| 0| 0| 0|516c4a0c|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:19.150|2017-01-11 20:57:19.150| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628f07d|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:19.172|2017-01-11 20:57:19.172| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628f5c8|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:19.172|2017-01-11 20:57:19.172| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628fb7c|00000000|000|000| 1| 1326| 0| 0| 0|idle
-2017-01-11 20:57:19.172|2017-01-11 20:57:19.172| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d8850|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:19.346|2017-01-11 20:57:19.346| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|06290082|00000000|000|000| 1| 1395| 0| 0| 0|idle
-2017-01-11 20:57:19.360|2017-01-11 20:57:19.360| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|062905cd|00000000|000|000| 1| 359| 0| 0| 0|idle
-2017-01-11 20:57:19.360|2017-01-11 20:57:19.360| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d8850|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:19.407|2017-01-11 20:57:19.407| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AP| 0| 0| 0|9b7a5a59|00000000|000|000| 1| 1017| 0| 0| 0|idle
-2017-01-11 20:57:19.512|2017-01-11 20:57:19.512| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| A| 0| 0| 0|63626f92|00000000|000|000| 1| 40| 0| 0| 0|idle
-2017-01-11 20:57:19.984|2017-01-11 20:57:19.984| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|3984446a|00000000|000|000| 1| 580| 0| 0| 0|idle
-2017-01-11 20:57:19.984|2017-01-11 20:57:19.984| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a887384a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:19.985|2017-01-11 20:57:19.985| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83687a76|00000000|000|000| 1| 629| 0| 0| 0|idle
-2017-01-11 20:57:19.985|2017-01-11 20:57:19.985| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a887384a|00000000|000|000| 1| 302| 0| 0| 0|idle
-2017-01-11 20:57:19.985|2017-01-11 20:57:19.985| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|3984467a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:19.993|2017-01-11 20:57:19.993| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df54c19|00000000|000|000| 1| 1032| 0| 0| 0|idle
-2017-01-11 20:57:19.993|2017-01-11 20:57:19.993| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687cb7|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:22.917|2017-01-11 20:57:22.917| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687cb7|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:22.917|2017-01-11 20:57:22.917| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|8368825f|00000000|000|000| 1| 908| 0| 0| 0|idle
-2017-01-11 20:57:22.917|2017-01-11 20:57:22.917| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df54fed|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:22.918|2017-01-11 20:57:22.918| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|3984467a|00000000|000|000| 1| 864| 0| 0| 0|idle
-2017-01-11 20:57:22.918|2017-01-11 20:57:22.918| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf49a|00000000|000|000| 1| 628| 0| 0| 0|idle
-2017-01-11 20:57:22.918|2017-01-11 20:57:22.918| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b817593|00000000|000|000| 1| 678| 0| 0| 0|idle
-2017-01-11 20:57:22.924|2017-01-11 20:57:22.924| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f503762b|00000000|000|000| 1| 544| 0| 0| 0|idle
-2017-01-11 20:57:22.924|2017-01-11 20:57:22.924| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817805|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:22.926|2017-01-11 20:57:22.926| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6aa0|00000000|000|000| 1| 501| 0| 0| 0|idle
-2017-01-11 20:57:22.926|2017-01-11 20:57:22.926| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bf6da|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:22.957|2017-01-11 20:57:22.957| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873944|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:22.983|2017-01-11 20:57:22.983| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df54fed|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:22.983|2017-01-11 20:57:22.983| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df55595|00000000|000|000| 1| 186| 0| 0| 0|idle
-2017-01-11 20:57:22.983|2017-01-11 20:57:22.983| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|836885b7|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:23.002|2017-01-11 20:57:23.002| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8873944|00000000|000|000| 1| 1067| 0| 0| 0|idle
-2017-01-11 20:57:23.003|2017-01-11 20:57:23.003| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|398449a6|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.618|2017-01-11 20:57:25.618| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|398449a6|00000000|000|000| 1| 504| 0| 0| 0|idle
-2017-01-11 20:57:25.618|2017-01-11 20:57:25.618| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873d3b|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.619|2017-01-11 20:57:25.619| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836885b7|00000000|000|000| 1| 836| 0| 0| 0|idle
-2017-01-11 20:57:25.619|2017-01-11 20:57:25.619| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf6da|00000000|000|000| 1| 608| 0| 0| 0|idle
-2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873d3b|00000000|000|000| 1| 2948| 0| 0| 0|idle
-2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a887488b|00000000|000|000| 1| 151| 0| 0| 0|idle
-2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39844b6a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39844b6a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39844b6a|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.658|2017-01-11 20:57:25.658| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df5561b|00000000|000|000| 1| 317| 0| 0| 0|idle
-2017-01-11 20:57:25.658|2017-01-11 20:57:25.658| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|836888c7|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.658|2017-01-11 20:57:25.658| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6c61|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:25.659|2017-01-11 20:57:25.659| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6c61|00000000|000|000| 1| 671| 0| 0| 0|idle
-2017-01-11 20:57:25.660|2017-01-11 20:57:25.660| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bf906|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:26.665|2017-01-11 20:57:26.665| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf906|00000000|000|000| 1| 580| 0| 0| 0|idle
-2017-01-11 20:57:26.665|2017-01-11 20:57:26.665| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6ecc|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:26.665|2017-01-11 20:57:26.665| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836888c7|00000000|000|000| 1| 629| 0| 0| 0|idle
-2017-01-11 20:57:26.666|2017-01-11 20:57:26.666| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6ecc|00000000|000|000| 1| 302| 0| 0| 0|idle
-2017-01-11 20:57:26.666|2017-01-11 20:57:26.666| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bfb16|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:26.672|2017-01-11 20:57:26.672| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df55724|00000000|000|000| 1| 1033| 0| 0| 0|idle
-2017-01-11 20:57:26.672|2017-01-11 20:57:26.672| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83688b08|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83688b08|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836890b0|00000000|000|000| 1| 908| 0| 0| 0|idle
-2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df55af9|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bfb16|00000000|000|000| 1| 864| 0| 0| 0|idle
-2017-01-11 20:57:29.859|2017-01-11 20:57:29.859| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6fc6|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:29.880|2017-01-11 20:57:29.880| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df55af9|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:29.880|2017-01-11 20:57:29.880| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df560a1|00000000|000|000| 1| 187| 0| 0| 0|idle
-2017-01-11 20:57:29.880|2017-01-11 20:57:29.880| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83689408|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:29.904|2017-01-11 20:57:29.904| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6fc6|00000000|000|000| 1| 1067| 0| 0| 0|idle
-2017-01-11 20:57:29.904|2017-01-11 20:57:29.904| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bfe42|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:30.749|2017-01-11 20:57:30.749| 0.000| 0.000| 17| 192.168.66.1| 5353| 224.0.0.251| 5353| 0| 0| 0| 0|00000000|00000000|000|000| 1| 68| 0| 0| 0|idle
-2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bfe42|00000000|000|000| 1| 1500| 0| 0| 0|idle
-2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c73bd|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53c03ea|00000000|000|000| 1| 136| 0| 0| 0|idle
-2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c73bd|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:30.874|2017-01-11 20:57:30.874| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c73bd|00000000|000|000| 1| 678| 0| 0| 0|idle
-2017-01-11 20:57:30.875|2017-01-11 20:57:30.875| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53c043e|00000000|000|000| 1| 52| 0| 0| 0|idle
-2017-01-11 20:57:31.755|2017-01-11 20:57:31.755| 0.000| 0.000| 17| 192.168.66.1| 5353| 224.0.0.251| 5353| 0| 0| 0| 0|00000000|00000000|000|000| 1| 68| 0| 0| 0|idle
-2017-01-11 20:57:32.919|2017-01-11 20:57:32.919| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53c043e|00000000|000|000| 1|
<TRUNCATED>
[06/50] [abbrv] metron git commit: METRON-1392 Fix a test case to
expect an Exception when replication factor more than number of brokers while
creating topic (MohanDV via merrimanr) closes apache/metron#892
Posted by rm...@apache.org.
METRON-1392 Fix a test case to expect an Exception when replication factor more than number of brokers while creating topic (MohanDV via merrimanr) closes apache/metron#892
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a421df22
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a421df22
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a421df22
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: a421df228b66d2100c29c2adee4248f35a70cafb
Parents: 24822dd
Author: MohanDV <mo...@gmail.com>
Authored: Fri Jan 26 08:21:51 2018 -0600
Committer: merrimanr <me...@apache.org>
Committed: Fri Jan 26 08:21:51 2018 -0600
----------------------------------------------------------------------
.../metron/rest/service/impl/KafkaServiceImplTest.java | 9 ++-------
1 file changed, 2 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a421df22/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
index 1f300ea..4527e8e 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
@@ -252,15 +252,10 @@ public class KafkaServiceImplTest {
@Test
public void createTopicShouldFailIfReplicationFactorIsGreaterThanAvailableBrokers() throws Exception {
- final Map<String, List<PartitionInfo>> topics = new HashMap<>();
-
- when(kafkaConsumer.listTopics()).thenReturn(topics);
-
+ exception.expect(RestException.class);
+ doThrow(AdminOperationException.class).when(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$));
kafkaService.createTopic(VALID_KAFKA_TOPIC);
- verify(adminUtils).createTopic(eq(zkUtils), eq("t"), eq(1), eq(2), eq(new Properties()), eq(RackAwareMode.Disabled$.MODULE$));
- verify(kafkaConsumer).listTopics();
- verifyZeroInteractions(zkUtils);
}
@Test
[45/50] [abbrv] metron git commit: METRON-1318 Update MacOS
Instructions for AWS (wardbekker via nickwallen) closes apache/metron#930
Posted by rm...@apache.org.
METRON-1318 Update MacOS Instructions for AWS (wardbekker via nickwallen) closes apache/metron#930
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/15b98dea
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/15b98dea
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/15b98dea
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 15b98dea48d9cab327f00a3a6a3e2117c0bbd214
Parents: fa86663
Author: wardbekker <wa...@wardbekker.com>
Authored: Sat Feb 10 13:23:16 2018 -0500
Committer: nickallen <ni...@apache.org>
Committed: Sat Feb 10 13:23:16 2018 -0500
----------------------------------------------------------------------
metron-deployment/amazon-ec2/README.md | 25 ++++++++++++++++++++++---
1 file changed, 22 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/15b98dea/metron-deployment/amazon-ec2/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/README.md b/metron-deployment/amazon-ec2/README.md
index f34632d..bc259ec 100644
--- a/metron-deployment/amazon-ec2/README.md
+++ b/metron-deployment/amazon-ec2/README.md
@@ -22,6 +22,15 @@ This project fully automates the provisioning of Apache Metron on Amazon EC2 inf
Warning: Amazon will charge for the use of their resources when running Apache Metron. The amount will vary based on the number and size of hosts, along with current Amazon pricing structure. Be sure to stop or terminate all of the hosts instantiated by Apache Metron when not in use to avoid unnecessary charges.
+AWS Defaults
+------------
+
+The Ansible playbook uses the following defaults for AWS deployment:
+
+- Instances: 10
+- Region: us-west-2
+- Instance type: m4.xlarge
+
Getting Started
---------------
@@ -41,11 +50,13 @@ Any platform that supports these tools is suitable, but the following instructio
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
```
-2. With Homebrew installed, run the following command in a terminal to install all of the required tools.
+2. With Homebrew installed, run the following command in a terminal to install all of the required tools and dependencies.
```
- brew cask install java
- brew install maven git
+ brew update
+ brew tap caskroom/versions
+ brew cask install java8 vagrant virtualbox
+ brew install maven git node
```
3. Install Ansible by following the instructions [here](http://docs.ansible.com/ansible/intro_installation.html#latest-releases-via-pip).
@@ -63,6 +74,14 @@ Any platform that supports these tools is suitable, but the following instructio
ssh-keygen -t rsa
```
+5. Ensure the JAVA_HOME environment variable is set
+
+ ```
+ export JAVA_HOME="/Library/Java/JavaVirtualMachines/jdk1.8.0_162.jdk/Contents/Home"
+ ```
+
+ Notice: You must replace the path with the installed JDK version path
+
### Amazon Web Services
If you already have an Amazon Web Services account that you have used to deploy EC2 hosts, then you should be able to skip the next few steps.
[39/50] [abbrv] metron git commit: METRON-1442: Split rest end points
for indexing topology into random access indexing and batch indexing this
closes apache/incubator-metron#923
Posted by rm...@apache.org.
METRON-1442: Split rest end points for indexing topology into random access indexing and batch indexing this closes apache/incubator-metron#923
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fcff0596
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fcff0596
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fcff0596
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: fcff0596c7d2b2546d89283fb90fbc8c10b31f1f
Parents: 0630505
Author: MohanDV <mo...@gmail.com>
Authored: Mon Feb 5 09:48:47 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Mon Feb 5 09:48:47 2018 -0500
----------------------------------------------------------------------
.../src/main/config/rest_application.yml | 3 +-
.../apache/metron/rest/MetronRestConstants.java | 6 +-
.../metron/rest/controller/StormController.java | 81 +++++++++++----
.../metron/rest/service/StormAdminService.java | 4 +-
.../service/impl/StormAdminServiceImpl.java | 8 +-
.../rest/service/impl/StormCLIWrapper.java | 16 +--
.../src/main/resources/application-test.yml | 3 +-
.../src/main/resources/application-vagrant.yml | 4 +-
.../StormControllerIntegrationTest.java | 102 ++++++++++---------
.../rest/mock/MockStormCLIClientWrapper.java | 93 ++++++++++++-----
.../metron/rest/mock/MockStormRestTemplate.java | 22 ++--
.../service/impl/StormAdminServiceImplTest.java | 8 +-
.../rest/service/impl/StormCLIWrapperTest.java | 17 ++--
13 files changed, 233 insertions(+), 134 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/config/rest_application.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml
index 6e4fb66..4cc51ff 100644
--- a/metron-interface/metron-rest/src/main/config/rest_application.yml
+++ b/metron-interface/metron-rest/src/main/config/rest_application.yml
@@ -46,7 +46,8 @@ storm:
enrichment:
script.path: ${METRON_HOME}/bin/start_enrichment_topology.sh
indexing:
- script.path: ${METRON_HOME}/bin/start_elasticsearch_topology.sh
+ randomaccess.script.path: ${METRON_HOME}/bin/start_elasticsearch_topology.sh
+ batch.script.path: ${METRON_HOME}/bin/start_hdfs_topology.sh
kerberos:
enabled: ${SECURITY_ENABLED}
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
index 4e8d7f2..f18d4cf 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java
@@ -35,10 +35,12 @@ public class MetronRestConstants {
public static final String TOPOLOGY_SUMMARY_URL = "/api/v1/topology/summary";
public static final String TOPOLOGY_URL = "/api/v1/topology";
public static final String ENRICHMENT_TOPOLOGY_NAME = "enrichment";
- public static final String INDEXING_TOPOLOGY_NAME = "indexing";
+ public static final String BATCH_INDEXING_TOPOLOGY_NAME = "batch_indexing";
+ public static final String RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME = "random_access_indexing";
public static final String PARSER_SCRIPT_PATH_SPRING_PROPERTY = "storm.parser.script.path";
public static final String ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY = "storm.enrichment.script.path";
- public static final String INDEXING_SCRIPT_PATH_SPRING_PROPERTY = "storm.indexing.script.path";
+ public static final String BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY = "storm.indexing.batch.script.path";
+ public static final String RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY = "storm.indexing.randomaccess.script.path";
public static final String PARSER_TOPOLOGY_OPTIONS_SPRING_PROPERTY = "storm.parser.topology.options";
public static final String KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY = "kafka.security.protocol";
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
index 292c668..d1af1c5 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/StormController.java
@@ -147,12 +147,12 @@ public class StormController {
return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME), HttpStatus.OK);
}
- @ApiOperation(value = "Retrieves the status of the Storm indexing topology")
+ @ApiOperation(value = "Retrieves the status of the Storm random access indexing topology")
@ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200),
@ApiResponse(message = "Topology is missing", code = 404) })
- @RequestMapping(value = "/indexing", method = RequestMethod.GET)
- ResponseEntity<TopologyStatus> getIndexing() throws RestException {
- TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(MetronRestConstants.INDEXING_TOPOLOGY_NAME);
+ @RequestMapping(value = "/indexing/randomaccess", method = RequestMethod.GET)
+ ResponseEntity<TopologyStatus> getRandomAccessIndexing() throws RestException {
+ TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME);
if (topologyStatus != null) {
return new ResponseEntity<>(topologyStatus, HttpStatus.OK);
} else {
@@ -160,32 +160,32 @@ public class StormController {
}
}
- @ApiOperation(value = "Starts a Storm indexing topology")
+ @ApiOperation(value = "Starts a Storm random access indexing topology")
@ApiResponse(message = "Returns start response message", code = 200)
- @RequestMapping(value = "/indexing/start", method = RequestMethod.GET)
- ResponseEntity<TopologyResponse> startIndexing() throws RestException {
- return new ResponseEntity<>(stormAdminService.startIndexingTopology(), HttpStatus.OK);
+ @RequestMapping(value = "/indexing/randomaccess/start", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> startRandomAccessIndexing() throws RestException {
+ return new ResponseEntity<>(stormAdminService.startIndexingTopology(MetronRestConstants.RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY), HttpStatus.OK);
}
- @ApiOperation(value = "Stops a Storm enrichment topology")
+ @ApiOperation(value = "Stops a Storm random access indexing topology")
@ApiResponse(message = "Returns stop response message", code = 200)
- @RequestMapping(value = "/indexing/stop", method = RequestMethod.GET)
- ResponseEntity<TopologyResponse> stopIndexing(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException {
- return new ResponseEntity<>(stormAdminService.stopIndexingTopology(stopNow), HttpStatus.OK);
+ @RequestMapping(value = "/indexing/randomaccess/stop", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> stopRandomAccessIndexing(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException {
+ return new ResponseEntity<>(stormAdminService.stopIndexingTopology(MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME ,stopNow), HttpStatus.OK);
}
- @ApiOperation(value = "Activates a Storm indexing topology")
+ @ApiOperation(value = "Activates a Storm random access indexing topology")
@ApiResponse(message = "Returns activate response message", code = 200)
- @RequestMapping(value = "/indexing/activate", method = RequestMethod.GET)
- ResponseEntity<TopologyResponse> activateIndexing() throws RestException {
- return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
+ @RequestMapping(value = "/indexing/randomaccess/activate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> activateRandomAccessIndexing() throws RestException {
+ return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
}
- @ApiOperation(value = "Deactivates a Storm indexing topology")
+ @ApiOperation(value = "Deactivates a Storm random access indexing topology")
@ApiResponse(message = "Returns deactivate response message", code = 200)
- @RequestMapping(value = "/indexing/deactivate", method = RequestMethod.GET)
- ResponseEntity<TopologyResponse> deactivateIndexing() throws RestException {
- return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
+ @RequestMapping(value = "/indexing/randomaccess/deactivate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> deactivateRandomAccessIndexing() throws RestException {
+ return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
}
@ApiOperation(value = "Retrieves information about the Storm command line client")
@@ -195,4 +195,45 @@ public class StormController {
return new ResponseEntity<>(stormAdminService.getStormClientStatus(), HttpStatus.OK);
}
+ @ApiOperation(value = "Retrieves the status of the Storm batch indexing topology")
+ @ApiResponses(value = { @ApiResponse(message = "Returns topology status information", code = 200),
+ @ApiResponse(message = "Topology is missing", code = 404) })
+ @RequestMapping(value = "/indexing/batch", method = RequestMethod.GET)
+ ResponseEntity<TopologyStatus> getBatchIndexing() throws RestException {
+ TopologyStatus topologyStatus = stormStatusService.getTopologyStatus(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME);
+ if (topologyStatus != null) {
+ return new ResponseEntity<>(topologyStatus, HttpStatus.OK);
+ } else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+ }
+ }
+
+ @ApiOperation(value = "Starts a Storm batch indexing topology")
+ @ApiResponse(message = "Returns start response message", code = 200)
+ @RequestMapping(value = "/indexing/batch/start", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> startBatchIndexing() throws RestException {
+ return new ResponseEntity<>(stormAdminService.startIndexingTopology(MetronRestConstants.BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Stops a Storm batch indexing topology")
+ @ApiResponse(message = "Returns stop response message", code = 200)
+ @RequestMapping(value = "/indexing/batch/stop", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> stopBatchIndexing(@ApiParam(name="stopNow", value="Stop the topology immediately")@RequestParam(required = false, defaultValue = "false") boolean stopNow) throws RestException {
+ return new ResponseEntity<>(stormAdminService.stopIndexingTopology(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME, stopNow), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Activates a Storm batch indexing topology")
+ @ApiResponse(message = "Returns activate response message", code = 200)
+ @RequestMapping(value = "/indexing/batch/activate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> activateBatchIndexing() throws RestException {
+ return new ResponseEntity<>(stormStatusService.activateTopology(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
+ }
+
+ @ApiOperation(value = "Deactivates a Storm batch indexing topology")
+ @ApiResponse(message = "Returns deactivate response message", code = 200)
+ @RequestMapping(value = "/indexing/batch/deactivate", method = RequestMethod.GET)
+ ResponseEntity<TopologyResponse> deactivateBatchIndexing() throws RestException {
+ return new ResponseEntity<>(stormStatusService.deactivateTopology(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME), HttpStatus.OK);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
index 8c1e228..3f6f8ff 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/StormAdminService.java
@@ -32,9 +32,9 @@ public interface StormAdminService {
TopologyResponse stopEnrichmentTopology(boolean stopNow) throws RestException;
- TopologyResponse startIndexingTopology() throws RestException;
+ TopologyResponse startIndexingTopology(String scriptPath) throws RestException;
- TopologyResponse stopIndexingTopology(boolean stopNow) throws RestException;
+ TopologyResponse stopIndexingTopology(String name, boolean stopNow) throws RestException;
Map<String, String> getStormClientStatus() throws RestException;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
index 9bd368f..40b01f1 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormAdminServiceImpl.java
@@ -74,13 +74,13 @@ public class StormAdminServiceImpl implements StormAdminService {
}
@Override
- public TopologyResponse startIndexingTopology() throws RestException {
- return createResponse(stormCLIClientWrapper.startIndexingTopology(), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR);
+ public TopologyResponse startIndexingTopology(String scriptPath) throws RestException {
+ return createResponse(stormCLIClientWrapper.startIndexingTopology(scriptPath), TopologyStatusCode.STARTED, TopologyStatusCode.START_ERROR);
}
@Override
- public TopologyResponse stopIndexingTopology(boolean stopNow) throws RestException {
- return createResponse(stormCLIClientWrapper.stopIndexingTopology(stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
+ public TopologyResponse stopIndexingTopology(String name, boolean stopNow) throws RestException {
+ return createResponse(stormCLIClientWrapper.stopIndexingTopology(name, stopNow), TopologyStatusCode.STOPPED, TopologyStatusCode.STOP_ERROR);
}
private TopologyResponse createResponse(int responseCode, TopologyStatusCode successMessage, TopologyStatusCode errorMessage) {
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
index fff7390..26049dd 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
@@ -37,7 +37,6 @@ import java.util.Map;
import static java.util.stream.Collectors.toList;
import static org.apache.metron.rest.MetronRestConstants.ENRICHMENT_TOPOLOGY_NAME;
-import static org.apache.metron.rest.MetronRestConstants.INDEXING_TOPOLOGY_NAME;
public class StormCLIWrapper {
@@ -70,14 +69,14 @@ public class StormCLIWrapper {
return runCommand(getStopCommand(ENRICHMENT_TOPOLOGY_NAME, stopNow));
}
- public int startIndexingTopology() throws RestException {
+ public int startIndexingTopology(String scriptPath) throws RestException {
kinit();
- return runCommand(getIndexingStartCommand());
+ return runCommand(getIndexingStartCommand(scriptPath));
}
- public int stopIndexingTopology(boolean stopNow) throws RestException {
+ public int stopIndexingTopology(String name, boolean stopNow) throws RestException {
kinit();
- return runCommand(getStopCommand(INDEXING_TOPOLOGY_NAME, stopNow));
+ return runCommand(getStopCommand(name, stopNow));
}
protected int runCommand(String[] command) throws RestException {
@@ -137,9 +136,9 @@ public class StormCLIWrapper {
return command;
}
- protected String[] getIndexingStartCommand() {
+ protected String[] getIndexingStartCommand(String scriptPath) {
String[] command = new String[1];
- command[0] = environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY);
+ command[0] = environment.getProperty(scriptPath);
return command;
}
@@ -166,7 +165,8 @@ public class StormCLIWrapper {
Map<String, String> status = new HashMap<>();
status.put("parserScriptPath", environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY));
status.put("enrichmentScriptPath", environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY));
- status.put("indexingScriptPath", environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY));
+ status.put("randomAccessIndexingScriptPath", environment.getProperty(MetronRestConstants.RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY));
+ status.put("batchIndexingScriptPath", environment.getProperty(MetronRestConstants.BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY));
status.put("stormClientVersionInstalled", stormClientVersionInstalled());
return status;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/resources/application-test.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-test.yml b/metron-interface/metron-rest/src/main/resources/application-test.yml
index 749dec4..3cca5e0 100644
--- a/metron-interface/metron-rest/src/main/resources/application-test.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-test.yml
@@ -38,7 +38,8 @@ storm:
enrichment:
script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh
indexing:
- script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
+ randomaccess.script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
+ batch.script.path: /usr/metron/${metron.version}/bin/start_hdfs_topology.sh
search:
max:
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
index cf2c170..3eea24a 100644
--- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
+++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml
@@ -49,7 +49,9 @@ storm:
enrichment:
script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh
indexing:
- script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
+ randomaccess.script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh
+ batch.script.path: /usr/metron/${metron.version}/bin/start_hdfs_topology.sh
+
kerberos:
enabled: false
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
index 9a6022c..3986413 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/StormControllerIntegrationTest.java
@@ -17,8 +17,10 @@
*/
package org.apache.metron.rest.controller;
+import org.apache.hadoop.hbase.shaded.com.google.common.collect.ImmutableList;
import org.apache.metron.common.configuration.SensorParserConfig;
import org.apache.metron.integration.utils.TestUtils;
+import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.model.TopologyStatusCode;
import org.apache.metron.rest.service.GlobalConfigService;
import org.apache.metron.rest.service.SensorParserConfigService;
@@ -33,6 +35,7 @@ import org.springframework.http.MediaType;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
import org.springframework.test.web.servlet.MockMvc;
+import org.springframework.test.web.servlet.ResultActions;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
@@ -287,66 +290,69 @@ public class StormControllerIntegrationTest {
.andExpect(jsonPath("$.status").value("SUCCESS"))
.andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
- this.mockMvc.perform(get(stormUrl + "/indexing").with(httpBasic(user,password)))
- .andExpect(status().isNotFound());
-
- this.mockMvc.perform(get(stormUrl + "/indexing/activate").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(jsonPath("$.status").value("ERROR"))
- .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
-
- this.mockMvc.perform(get(stormUrl + "/indexing/deactivate").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(jsonPath("$.status").value("ERROR"))
- .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
-
- this.mockMvc.perform(get(stormUrl + "/indexing/stop?stopNow=true").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(jsonPath("$.status").value("ERROR"))
- .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
-
- this.mockMvc.perform(get(stormUrl + "/indexing/start").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(jsonPath("$.status").value("SUCCESS"))
- .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
-
- this.mockMvc.perform(get(stormUrl + "/indexing/deactivate").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(jsonPath("$.status").value("SUCCESS"))
- .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
-
- this.mockMvc.perform(get(stormUrl + "/indexing/activate").with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(jsonPath("$.status").value("SUCCESS"))
- .andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
-
- this.mockMvc.perform(get(stormUrl + "/indexing").with(httpBasic(user,password)))
+ for(String type : ImmutableList.of("randomaccess", "batch")) {
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type).with(httpBasic(user,password)))
+ .andExpect(status().isNotFound());
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/activate").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/deactivate").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.TOPOLOGY_NOT_FOUND.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/stop?stopNow=true").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("ERROR"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STOP_ERROR.toString()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/start").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.STARTED.toString()));
+
+ ResultActions actions = this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/deactivate").with(httpBasic(user, password)));
+ actions.andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.INACTIVE.name()));
+
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/activate").with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(jsonPath("$.status").value("SUCCESS"))
+ .andExpect(jsonPath("$.message").value(TopologyStatusCode.ACTIVE.name()));
+ String topologyName = type.equals("randomaccess")? MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME:MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME;
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type).with(httpBasic(user, password)))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
+ .andExpect(jsonPath("$.name").value(topologyName))
+ .andExpect(jsonPath("$.id", containsString("indexing")))
+ .andExpect(jsonPath("$.status").value("ACTIVE"))
+ .andExpect(jsonPath("$.latency").exists())
+ .andExpect(jsonPath("$.throughput").exists())
+ .andExpect(jsonPath("$.emitted").exists())
+ .andExpect(jsonPath("$.acked").exists());
+ this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$.name").value("indexing"))
- .andExpect(jsonPath("$.id", containsString("indexing")))
- .andExpect(jsonPath("$.status").value("ACTIVE"))
- .andExpect(jsonPath("$.latency").exists())
- .andExpect(jsonPath("$.throughput").exists())
- .andExpect(jsonPath("$.emitted").exists())
- .andExpect(jsonPath("$.acked").exists());
+ .andExpect(jsonPath("$[?(@.name == '" + topologyName + "' && @.status == 'ACTIVE')]").exists());
- this.mockMvc.perform(get(stormUrl).with(httpBasic(user,password)))
- .andExpect(status().isOk())
- .andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$[?(@.name == 'indexing' && @.status == 'ACTIVE')]").exists());
-
- this.mockMvc.perform(get(stormUrl + "/indexing/stop").with(httpBasic(user,password)))
+ this.mockMvc.perform(get(stormUrl + "/indexing/" + type + "/stop").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(jsonPath("$.status").value("SUCCESS"))
.andExpect(jsonPath("$.message").value(TopologyStatusCode.STOPPED.name()));
+ }
+
+
this.mockMvc.perform(get(stormUrl + "/client/status").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(jsonPath("$.stormClientVersionInstalled").value("1.0.1"))
.andExpect(jsonPath("$.parserScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_parser_topology.sh"))
.andExpect(jsonPath("$.enrichmentScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_enrichment_topology.sh"))
- .andExpect(jsonPath("$.indexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh"));
+ .andExpect(jsonPath("$.randomAccessIndexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_elasticsearch_topology.sh"))
+ .andExpect(jsonPath("$.batchIndexingScriptPath").value("/usr/metron/" + metronVersion + "/bin/start_hdfs_topology.sh"));
globalConfigService.delete();
sensorParserConfigService.delete("broTest");
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
index dd21095..9018935 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormCLIClientWrapper.java
@@ -17,6 +17,7 @@
*/
package org.apache.metron.rest.mock;
+import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.TopologyStatusCode;
import org.apache.metron.rest.service.impl.StormCLIWrapper;
@@ -29,7 +30,8 @@ public class MockStormCLIClientWrapper extends StormCLIWrapper {
private final Map<String, TopologyStatusCode> parsersStatus = new HashMap<>();
private TopologyStatusCode enrichmentStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
- private TopologyStatusCode indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ private TopologyStatusCode randomAccessIndexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ private TopologyStatusCode batchIndexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
public Set<String> getParserTopologyNames() {
return parsersStatus.keySet();
@@ -128,45 +130,84 @@ public class MockStormCLIClientWrapper extends StormCLIWrapper {
}
}
- public TopologyStatusCode getIndexingStatus() {
- return indexingStatus;
+ public TopologyStatusCode getIndexingStatus(String name) {
+ return name.equals(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME)?batchIndexingStatus:randomAccessIndexingStatus;
}
@Override
- public int startIndexingTopology() throws RestException {
- if (indexingStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
- indexingStatus = TopologyStatusCode.ACTIVE;
- return 0;
- } else {
- return 1;
+ public int startIndexingTopology(String scriptPath) throws RestException {
+ if(scriptPath.equals(MetronRestConstants.BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY)) {
+ if (batchIndexingStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ batchIndexingStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ else {
+ if (randomAccessIndexingStatus == TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ randomAccessIndexingStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
}
}
@Override
- public int stopIndexingTopology(boolean stopNow) throws RestException {
- if (indexingStatus == TopologyStatusCode.ACTIVE) {
- indexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
- return 0;
- } else {
- return 1;
+ public int stopIndexingTopology(String name, boolean stopNow) throws RestException {
+ if(name.equals(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME)) {
+ if (batchIndexingStatus == TopologyStatusCode.ACTIVE) {
+ batchIndexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ else {
+ if (randomAccessIndexingStatus == TopologyStatusCode.ACTIVE) {
+ randomAccessIndexingStatus = TopologyStatusCode.TOPOLOGY_NOT_FOUND;
+ return 0;
+ } else {
+ return 1;
+ }
}
}
- public int activateIndexingTopology() {
- if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) {
- indexingStatus = TopologyStatusCode.ACTIVE;
- return 0;
- } else {
- return 1;
+ public int activateIndexingTopology(String name) {
+ if(name.equals(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME)) {
+ if (batchIndexingStatus == TopologyStatusCode.INACTIVE || batchIndexingStatus == TopologyStatusCode.ACTIVE) {
+ batchIndexingStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ else {
+ if (randomAccessIndexingStatus == TopologyStatusCode.INACTIVE || randomAccessIndexingStatus == TopologyStatusCode.ACTIVE) {
+ randomAccessIndexingStatus = TopologyStatusCode.ACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
}
}
- public int deactivateIndexingTopology() {
- if (indexingStatus == TopologyStatusCode.INACTIVE || indexingStatus == TopologyStatusCode.ACTIVE) {
- indexingStatus = TopologyStatusCode.INACTIVE;
- return 0;
+ public int deactivateIndexingTopology(String name) {
+ if (name.equals(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME)) {
+ if (batchIndexingStatus == TopologyStatusCode.INACTIVE || batchIndexingStatus == TopologyStatusCode.ACTIVE) {
+ batchIndexingStatus = TopologyStatusCode.INACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
} else {
- return 1;
+ if (randomAccessIndexingStatus == TopologyStatusCode.INACTIVE || randomAccessIndexingStatus == TopologyStatusCode.ACTIVE) {
+ randomAccessIndexingStatus = TopologyStatusCode.INACTIVE;
+ return 0;
+ } else {
+ return 1;
+ }
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
index ccf993d..ef47ac9 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/mock/MockStormRestTemplate.java
@@ -53,9 +53,13 @@ public class MockStormRestTemplate extends RestTemplate {
if (enrichmentStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
topologyStatusList.add(getTopologyStatus("enrichment"));
}
- TopologyStatusCode indexingStatus = mockStormCLIClientWrapper.getIndexingStatus();
- if (indexingStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
- topologyStatusList.add(getTopologyStatus("indexing"));
+ TopologyStatusCode batchIndexingStatus = mockStormCLIClientWrapper.getIndexingStatus(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME);
+ if (batchIndexingStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ topologyStatusList.add(getTopologyStatus(MetronRestConstants.BATCH_INDEXING_TOPOLOGY_NAME));
+ }
+ TopologyStatusCode randomIndexingStatus = mockStormCLIClientWrapper.getIndexingStatus(MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME);
+ if (randomIndexingStatus != TopologyStatusCode.TOPOLOGY_NOT_FOUND) {
+ topologyStatusList.add(getTopologyStatus(MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME));
}
topologySummary.setTopologies(topologyStatusList.toArray(new TopologyStatus[topologyStatusList.size()]));
response = topologySummary;
@@ -79,8 +83,8 @@ public class MockStormRestTemplate extends RestTemplate {
topologyStatus.setId(name + "-id");
if ("enrichment".equals(name)) {
topologyStatus.setStatus(mockStormCLIClientWrapper.getEnrichmentStatus());
- } else if ("indexing".equals(name)) {
- topologyStatus.setStatus(mockStormCLIClientWrapper.getIndexingStatus());
+ } else if (name.contains("indexing")) {
+ topologyStatus.setStatus(mockStormCLIClientWrapper.getIndexingStatus(name));
} else {
topologyStatus.setStatus(mockStormCLIClientWrapper.getParserStatus(name));
}
@@ -97,16 +101,16 @@ public class MockStormRestTemplate extends RestTemplate {
if (action.equals("activate")) {
if (name.equals("enrichment")) {
returnCode = mockStormCLIClientWrapper.activateEnrichmentTopology();
- } else if (name.equals("indexing")) {
- returnCode = mockStormCLIClientWrapper.activateIndexingTopology();
+ } else if (name.contains("indexing")) {
+ returnCode = mockStormCLIClientWrapper.activateIndexingTopology(name);
} else {
returnCode = mockStormCLIClientWrapper.activateParserTopology(name);
}
} else if (action.equals("deactivate")){
if (name.equals("enrichment")) {
returnCode = mockStormCLIClientWrapper.deactivateEnrichmentTopology();
- } else if (name.equals("indexing")) {
- returnCode = mockStormCLIClientWrapper.deactivateIndexingTopology();
+ } else if (name.contains("indexing")) {
+ returnCode = mockStormCLIClientWrapper.deactivateIndexingTopology(name);
} else {
returnCode = mockStormCLIClientWrapper.deactivateParserTopology(name);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
index d83a74c..65a1bda 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormAdminServiceImplTest.java
@@ -122,22 +122,22 @@ public class StormAdminServiceImplTest {
@Test
public void startIndexingTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
- when(stormCLIClientWrapper.startIndexingTopology()).thenReturn(0);
+ when(stormCLIClientWrapper.startIndexingTopology("random_access_indexing_script_path")).thenReturn(0);
TopologyResponse expected = new TopologyResponse();
expected.setSuccessMessage(TopologyStatusCode.STARTED.toString());
- assertEquals(expected, stormAdminService.startIndexingTopology());
+ assertEquals(expected, stormAdminService.startIndexingTopology("random_access_indexing_script_path"));
}
@Test
public void stopIndexingTopologyShouldProperlyReturnSuccessTopologyResponse() throws Exception {
- when(stormCLIClientWrapper.stopIndexingTopology(false)).thenReturn(0);
+ when(stormCLIClientWrapper.stopIndexingTopology("random_access_indexing", false)).thenReturn(0);
TopologyResponse expected = new TopologyResponse();
expected.setSuccessMessage(TopologyStatusCode.STOPPED.toString());
- assertEquals(expected, stormAdminService.stopIndexingTopology(false));
+ assertEquals(expected, stormAdminService.stopIndexingTopology("random_access_indexing",false));
}
@Test
http://git-wip-us.apache.org/repos/asf/metron/blob/fcff0596/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
index 73d54d8..60a9790 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/StormCLIWrapperTest.java
@@ -174,11 +174,11 @@ public class StormCLIWrapperTest {
whenNew(ProcessBuilder.class).withParameterTypes(String[].class).withArguments(anyVararg()).thenReturn(processBuilder);
when(processBuilder.start()).thenReturn(process);
- when(environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing");
+ when(environment.getProperty(MetronRestConstants.RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing");
when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(false);
when(process.exitValue()).thenReturn(0);
- assertEquals(0, stormCLIWrapper.startIndexingTopology());
+ assertEquals(0, stormCLIWrapper.startIndexingTopology(MetronRestConstants.RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY));
verify(process).waitFor();
verifyNew(ProcessBuilder.class).withArguments("/start_indexing");
@@ -192,9 +192,9 @@ public class StormCLIWrapperTest {
when(environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)).thenReturn(false);
when(process.exitValue()).thenReturn(0);
- assertEquals(0, stormCLIWrapper.stopIndexingTopology(false));
+ assertEquals(0, stormCLIWrapper.stopIndexingTopology("random_access_indexing", false));
verify(process).waitFor();
- verifyNew(ProcessBuilder.class).withArguments("storm", "kill", MetronRestConstants.INDEXING_TOPOLOGY_NAME);
+ verifyNew(ProcessBuilder.class).withArguments("storm", "kill", MetronRestConstants.RANDOM_ACCESS_INDEXING_TOPOLOGY_NAME);
}
@Test
@@ -209,15 +209,16 @@ public class StormCLIWrapperTest {
when(process.getInputStream()).thenReturn(inputStream);
when(environment.getProperty(MetronRestConstants.PARSER_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_parser");
when(environment.getProperty(MetronRestConstants.ENRICHMENT_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_enrichment");
- when(environment.getProperty(MetronRestConstants.INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_indexing");
-
+ when(environment.getProperty(MetronRestConstants.RANDOM_ACCESS_INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_elasticsearch");
+ when(environment.getProperty(MetronRestConstants.BATCH_INDEXING_SCRIPT_PATH_SPRING_PROPERTY)).thenReturn("/start_hdfs");
Map<String, String> actual = stormCLIWrapper.getStormClientStatus();
assertEquals(new HashMap<String, String>() {{
- put("parserScriptPath", "/start_parser");
+ put("randomAccessIndexingScriptPath", "/start_elasticsearch");
put("enrichmentScriptPath", "/start_enrichment");
- put("indexingScriptPath", "/start_indexing");
put("stormClientVersionInstalled", "1.1");
+ put("parserScriptPath", "/start_parser");
+ put("batchIndexingScriptPath", "/start_hdfs");
}}, actual);
verifyNew(ProcessBuilder.class).withArguments("storm", "version");
[40/50] [abbrv] metron git commit: METRON-1446: Fix openjdk issue
with Ubuntu (mmiklavc via mmiklavc) closes apache/metron#926
Posted by rm...@apache.org.
METRON-1446: Fix openjdk issue with Ubuntu (mmiklavc via mmiklavc) closes apache/metron#926
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/38b7d5e3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/38b7d5e3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/38b7d5e3
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 38b7d5e339320e71cd4d35d31b8d118a3e6d0be7
Parents: fcff059
Author: mmiklavc <mi...@gmail.com>
Authored: Tue Feb 6 09:57:15 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Feb 6 09:57:15 2018 -0700
----------------------------------------------------------------------
.../ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/38b7d5e3/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
index 8337b81..4eb0945 100644
--- a/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
@@ -20,7 +20,7 @@
register: jdk_dir
- name: Install openjdk repository
- shell: add-apt-repository ppa:openjdk-r/ppa
+ shell: add-apt-repository ppa:openjdk-r/ppa -y
when: not jdk_dir.stat.exists
- name: Update package cache
[27/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
METRON-1370 Create Full Dev Equivalent for Ubuntu (nickwallen via cestella) closes apache/incubator-metron#903
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6f267991
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6f267991
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6f267991
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 6f2679919a9e6c9f3a146d52b10c6dbe2ce55180
Parents: 878003a
Author: nickwallen <ni...@nickallen.org>
Authored: Fri Jan 26 15:56:01 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Fri Jan 26 15:56:01 2018 -0500
----------------------------------------------------------------------
README.md | 32 +-
dev-utilities/release-utils/metron-rc-check | 2 +-
metron-contrib/metron-docker/README.md | 14 +-
metron-deployment/Kerberos-manual-setup.md | 4 +-
metron-deployment/README.md | 2 +-
metron-deployment/amazon-ec2/ansible.cfg | 4 +-
metron-deployment/amazon-ec2/conf/defaults.yml | 8 -
metron-deployment/amazon-ec2/playbook.yml | 2 +-
metron-deployment/ansible/README.md | 24 +
.../extra_modules/ambari_cluster_state.py | 395 +
.../extra_modules/ambari_service_state.py | 352 +
.../ansible/playbooks/ambari_install.yml | 54 +
.../ansible/playbooks/docker_probe_install.yml | 62 +
.../ansible/playbooks/metron_build.yml | 23 +
.../ansible/playbooks/metron_full_install.yml | 46 +
.../ansible/playbooks/metron_install.yml | 75 +
.../ansible/playbooks/sensor_install.yml | 59 +
metron-deployment/ansible/roles/README.md | 24 +
.../roles/ambari_common/defaults/main.yml | 23 +
.../ansible/roles/ambari_common/meta/main.yml | 21 +
.../ambari_common/tasks/ambari-repo-centos.yml | 22 +
.../ambari_common/tasks/ambari-repo-ubuntu.yml | 23 +
.../ambari_common/tasks/elasticsearch-repo.yml | 40 +
.../roles/ambari_common/tasks/hostname.yml | 32 +
.../ambari_common/tasks/iptables-centos.yml | 20 +
.../ambari_common/tasks/iptables-ubuntu.yml | 19 +
.../roles/ambari_common/tasks/logrotate.yml | 22 +
.../ansible/roles/ambari_common/tasks/main.yml | 39 +
.../roles/ambari_common/tasks/nodejs.yml | 34 +
.../templates/metron-hadoop-logrotate.yml | 149 +
.../roles/ambari_config/defaults/main.yml | 38 +
.../ansible/roles/ambari_config/meta/main.yml | 22 +
.../roles/ambari_config/tasks/dependencies.yml | 24 +
.../ansible/roles/ambari_config/tasks/main.yml | 39 +
.../roles/ambari_config/vars/single_node_vm.yml | 135 +
.../roles/ambari_config/vars/small_cluster.yml | 149 +
.../roles/ambari_gather_facts/meta/main.yml | 21 +
.../roles/ambari_gather_facts/tasks/main.yml | 234 +
.../roles/ambari_master/defaults/main.yml | 21 +
.../roles/ambari_master/tasks/ambari.yml | 36 +
.../ansible/roles/ambari_master/tasks/main.yml | 24 +
.../ansible/roles/ambari_master/tasks/mpack.yml | 26 +
.../roles/ambari_slave/defaults/main.yml | 18 +
.../roles/ambari_slave/files/hostname.sh | 19 +
.../ansible/roles/ambari_slave/meta/main.yml | 19 +
.../roles/ambari_slave/tasks/install-agent.yml | 24 +
.../ansible/roles/ambari_slave/tasks/main.yml | 46 +
.../ansible/roles/bro/meta/main.yml | 22 +
.../ansible/roles/bro/tasks/bro.yml | 59 +
.../ansible/roles/bro/tasks/dependencies.yml | 48 +
.../ansible/roles/bro/tasks/librdkafka.yml | 39 +
.../ansible/roles/bro/tasks/main.yml | 22 +
.../roles/bro/tasks/metron-bro-plugin-kafka.yml | 49 +
.../ansible/roles/bro/tasks/nic.yml | 19 +
.../ansible/roles/bro/vars/main.yml | 34 +
.../ansible/roles/build-tools/meta/main.yml | 19 +
.../ansible/roles/build-tools/tasks/main.yml | 34 +
.../roles/deployment-report/defaults/main.yml | 19 +
.../roles/deployment-report/meta/main.yml | 19 +
.../roles/deployment-report/tasks/main.yml | 32 +
.../ansible/roles/enable-swap/defaults/main.yml | 19 +
.../roles/enable-swap/tasks/enable-swap.yml | 35 +
.../ansible/roles/enable-swap/tasks/main.yml | 31 +
.../ansible/roles/epel/tasks/main.yml | 20 +
.../ansible/roles/fastcapa/defaults/main.yml | 42 +
.../ansible/roles/fastcapa/meta/main.yml | 19 +
.../ansible/roles/fastcapa/tasks/debug.yml | 26 +
.../roles/fastcapa/tasks/dependencies.yml | 39 +
.../ansible/roles/fastcapa/tasks/dpdk.yml | 59 +
.../ansible/roles/fastcapa/tasks/fastcapa.yml | 46 +
.../ansible/roles/fastcapa/tasks/kernel.yml | 51 +
.../ansible/roles/fastcapa/tasks/main.yml | 22 +
.../ansible/roles/fastcapa/templates/fastcapa | 145 +
.../roles/fastcapa/templates/fastcapa.conf | 67 +
.../ansible/roles/httplib2/tasks/main.yml | 20 +
.../ansible/roles/java_jdk/defaults/main.yml | 18 +
.../ansible/roles/java_jdk/tasks/main.yml | 34 +
.../roles/kafka-broker/defaults/main.yml | 18 +
.../ansible/roles/kafka-broker/meta/main.yml | 18 +
.../ansible/roles/kafka-broker/tasks/main.yml | 41 +
.../roles/kafka-client/defaults/main.yml | 18 +
.../ansible/roles/kafka-client/tasks/main.yml | 30 +
.../ansible/roles/librdkafka/defaults/main.yml | 20 +
.../roles/librdkafka/tasks/dependencies.yml | 40 +
.../roles/librdkafka/tasks/librdkafka.yml | 39 +
.../ansible/roles/librdkafka/tasks/main.yml | 19 +
.../roles/libselinux-python/tasks/main.yml | 26 +
.../roles/load_web_templates/meta/main.yml | 19 +
.../roles/load_web_templates/tasks/main.yml | 24 +
.../roles/metron-builder/defaults/main.yml | 18 +
.../roles/metron-builder/tasks/build-debs.yml | 26 +
.../roles/metron-builder/tasks/build-rpms.yml | 26 +
.../roles/metron-builder/tasks/build.yml | 26 +
.../roles/metron-builder/tasks/clean.yml | 32 +
.../ansible/roles/metron-builder/tasks/main.yml | 35 +
.../roles/metron-packages/defaults/main.yml | 21 +
.../tasks/copy_packages_centos.yml | 25 +
.../tasks/copy_packages_ubuntu.yml | 31 +
.../metron-packages/tasks/create_directory.yml | 27 +
.../tasks/create_repo_centos.yml | 24 +
.../tasks/create_repo_ubuntu.yml | 32 +
.../roles/metron-packages/tasks/main.yml | 52 +
.../roles/metron-packages/tasks/validate.yml | 24 +
.../files/es_templates/error_index.template | 57 +
.../roles/metron_pcapservice/defaults/main.yml | 28 +
.../roles/metron_pcapservice/meta/main.yml | 19 +
.../metron_pcapservice/tasks/config-hbase.yml | 26 +
.../roles/metron_pcapservice/tasks/main.yml | 25 +
.../metron_pcapservice/tasks/pcapservice.yml | 30 +
.../templates/metron-pcapservice-logrotate.yml | 27 +
.../metron_pcapservice/templates/pcapservice | 84 +
.../ansible/roles/ntp/tasks/main.yml | 37 +
.../ansible/roles/opentaxii/README.md | 178 +
.../ansible/roles/opentaxii/defaults/main.yml | 43 +
.../ansible/roles/opentaxii/meta/main.yml | 17 +
.../roles/opentaxii/tasks/dependencies.yml | 37 +
.../roles/opentaxii/tasks/hailataxii.yml | 45 +
.../ansible/roles/opentaxii/tasks/main.yml | 20 +
.../ansible/roles/opentaxii/tasks/opentaxii.yml | 50 +
.../opentaxii/templates/collection-status.py | 32 +
.../roles/opentaxii/templates/collections.yml | 19 +
.../ansible/roles/opentaxii/templates/opentaxii | 176 +
.../opentaxii/templates/opentaxii-conf.yml | 38 +
.../roles/opentaxii/templates/services.yml | 69 +
.../ansible/roles/pcap_replay/README.md | 61 +
.../ansible/roles/pcap_replay/defaults/main.yml | 21 +
.../ansible/roles/pcap_replay/meta/main.yml | 17 +
.../roles/pcap_replay/tasks/dependencies.yml | 32 +
.../ansible/roles/pcap_replay/tasks/main.yml | 20 +
.../ansible/roles/pcap_replay/tasks/service.yml | 25 +
.../roles/pcap_replay/tasks/tcpreplay.yml | 38 +
.../roles/pcap_replay/templates/pcap-replay | 97 +
.../ansible/roles/pycapa/defaults/main.yml | 26 +
.../ansible/roles/pycapa/meta/main.yml | 20 +
.../ansible/roles/pycapa/tasks/dependencies.yml | 36 +
.../ansible/roles/pycapa/tasks/main.yml | 21 +
.../roles/pycapa/tasks/pycapa-service.yml | 22 +
.../ansible/roles/pycapa/tasks/pycapa.yml | 40 +
.../ansible/roles/pycapa/templates/pycapa | 115 +
.../ansible/roles/python-pip/tasks/main.yml | 24 +
.../ansible/roles/sensor-stubs/README.md | 99 +
.../roles/sensor-stubs/defaults/main.yml | 25 +
.../ansible/roles/sensor-stubs/files/bro.out | 1346 +
.../ansible/roles/sensor-stubs/files/snort.out | 27404 +++++++++++++++++
.../ansible/roles/sensor-stubs/files/yaf.out | 22164 +++++++++++++
.../ansible/roles/sensor-stubs/tasks/main.yml | 43 +
.../roles/sensor-stubs/templates/sensor-stubs | 154 +
.../roles/sensor-stubs/templates/start-bro-stub | 55 +
.../sensor-stubs/templates/start-snort-stub | 55 +
.../roles/sensor-stubs/templates/start-yaf-stub | 56 +
.../ansible/roles/sensor-test-mode/README.md | 44 +
.../roles/sensor-test-mode/defaults/main.yml | 20 +
.../roles/sensor-test-mode/files/example.pcap | Bin 0 -> 507865 bytes
.../roles/sensor-test-mode/meta/main.yml | 19 +
.../roles/sensor-test-mode/tasks/main.yml | 25 +
.../roles/sensor-test-mode/tasks/pcap.yml | 22 +
.../roles/sensor-test-mode/tasks/snort.yml | 36 +
.../roles/sensor-test-mode/tasks/yaf.yml | 30 +
.../ansible/roles/snort/defaults/main.yml | 28 +
.../ansible/roles/snort/files/snort.conf | 730 +
.../ansible/roles/snort/meta/main.yml | 23 +
.../ansible/roles/snort/tasks/daq.yml | 36 +
.../ansible/roles/snort/tasks/main.yml | 21 +
.../ansible/roles/snort/tasks/nic.yml | 19 +
.../ansible/roles/snort/tasks/producer.yml | 31 +
.../ansible/roles/snort/tasks/snort.yml | 85 +
.../roles/snort/templates/snort-producer | 83 +
.../snort/templates/start-snort-producer.sh | 24 +
.../roles/tap_interface/defaults/main.yml | 19 +
.../ansible/roles/tap_interface/tasks/main.yml | 35 +
.../ansible/roles/yaf/defaults/main.yml | 29 +
.../ansible/roles/yaf/meta/main.yml | 22 +
.../ansible/roles/yaf/tasks/fixbuf.yml | 37 +
.../ansible/roles/yaf/tasks/main.yml | 20 +
.../ansible/roles/yaf/tasks/nic.yml | 19 +
.../ansible/roles/yaf/tasks/yaf.yml | 53 +
.../ansible/roles/yaf/templates/start-yaf.sh | 25 +
.../ansible/roles/yaf/templates/yaf | 83 +
metron-deployment/development/README.md | 24 +
.../development/centos6/.gitignore | 1 +
metron-deployment/development/centos6/README.md | 105 +
.../development/centos6/Vagrantfile | 92 +
.../development/centos6/ansible.cfg | 27 +
.../centos6/ansible/inventory/group_vars/all | 85 +
.../development/centos6/ansible/inventory/hosts | 47 +
.../development/fastcapa/.gitignore | 1 +
.../development/fastcapa/README.md | 138 +
.../development/fastcapa/centos-7.1/Vagrantfile | 72 +
.../development/fastcapa/centos-7.1/ansible.cfg | 22 +
.../fastcapa/centos-7.1/vars/main.yml | 52 +
.../development/fastcapa/centos-7.4/Vagrantfile | 72 +
.../development/fastcapa/centos-7.4/ansible.cfg | 22 +
.../fastcapa/centos-7.4/vars/main.yml | 52 +
.../development/fastcapa/playbook.yml | 49 +
.../tasks/validate-messages-received.yml | 38 +
.../fastcapa/tasks/validate-packets-sent.yml | 41 +
.../development/ubuntu14/README.md | 106 +
.../development/ubuntu14/Vagrantfile | 87 +
.../development/ubuntu14/ansible.cfg | 27 +
.../ubuntu14/ansible/inventory/group_vars/all | 85 +
.../ubuntu14/ansible/inventory/hosts | 47 +
.../development/ubuntu14/ansible/playbook.yml | 25 +
.../extra_modules/ambari_cluster_state.py | 395 -
.../extra_modules/ambari_service_state.py | 352 -
.../inventory/full-dev-platform/group_vars/all | 89 -
.../inventory/full-dev-platform/hosts | 47 -
.../METRON/CURRENT/repos/repoinfo.xml | 8 +-
.../5.6.2/configuration/elastic-sysconfig.xml | 7 +-
.../5.6.2/configuration/elastic-systemd.xml | 30 +
.../ELASTICSEARCH/5.6.2/metainfo.xml | 19 +-
.../5.6.2/package/scripts/elastic.py | 86 -
.../5.6.2/package/scripts/elastic_commands.py | 253 +
.../5.6.2/package/scripts/elastic_master.py | 49 +-
.../5.6.2/package/scripts/elastic_slave.py | 55 +-
.../5.6.2/package/scripts/params.py | 13 +
.../5.6.2/package/scripts/service_check.py | 80 +-
.../5.6.2/package/scripts/slave.py | 80 -
.../5.6.2/package/scripts/status_params.py | 1 +
.../common-services/KIBANA/5.6.2/metainfo.xml | 21 +-
.../KIBANA/5.6.2/package/scripts/common.py | 56 +
.../5.6.2/package/scripts/kibana_master.py | 56 +-
.../CURRENT/package/scripts/indexing_master.py | 3 +-
.../packaging/docker/deb-docker/build.sh | 59 +-
.../docker/deb-docker/debian/changelog | 22 -
.../packaging/docker/deb-docker/debian/control | 28 -
.../docker/deb-docker/debian/copyright | 35 -
.../deb-docker/debian/metron-alerts/postinst | 26 +
.../deb-docker/debian/metron-alerts/postrm | 25 +
.../deb-docker/debian/metron-alerts/preinst | 29 +
.../deb-docker/debian/metron-alerts/prerm | 30 +
.../deb-docker/debian/metron-config/postinst | 26 +
.../deb-docker/debian/metron-config/postrm | 25 +
.../deb-docker/debian/metron-config/preinst | 29 +
.../deb-docker/debian/metron-config/prerm | 30 +
.../docker/deb-docker/debian/metron/changelog | 28 +
.../docker/deb-docker/debian/metron/control | 34 +
.../docker/deb-docker/debian/metron/copyright | 41 +
.../packaging/docker/deb-docker/pom.xml | 6 +
.../packaging/docker/rpm-docker/pom.xml | 6 +
metron-deployment/playbooks/ambari_install.yml | 61 -
.../playbooks/docker_probe_install.yml | 62 -
metron-deployment/playbooks/metron_build.yml | 23 -
.../playbooks/metron_full_install.yml | 34 -
metron-deployment/playbooks/metron_install.yml | 102 -
metron-deployment/roles/README.md | 24 -
.../roles/ambari_common/defaults/main.yml | 20 -
.../roles/ambari_common/meta/main.yml | 21 -
.../roles/ambari_common/tasks/main.yml | 54 -
.../templates/metron-hadoop-logrotate.yml | 149 -
.../roles/ambari_config/defaults/main.yml | 38 -
.../roles/ambari_config/meta/main.yml | 22 -
.../roles/ambari_config/tasks/main.yml | 46 -
.../roles/ambari_config/vars/single_node_vm.yml | 135 -
.../roles/ambari_config/vars/small_cluster.yml | 149 -
.../roles/ambari_gather_facts/meta/main.yml | 21 -
.../roles/ambari_gather_facts/tasks/main.yml | 234 -
.../roles/ambari_master/defaults/main.yml | 19 -
.../roles/ambari_master/tasks/main.yml | 52 -
.../roles/ambari_slave/defaults/main.yml | 18 -
.../roles/ambari_slave/files/hostname.sh | 19 -
.../roles/ambari_slave/meta/main.yml | 19 -
.../roles/ambari_slave/tasks/main.yml | 56 -
metron-deployment/roles/bro/meta/main.yml | 22 -
metron-deployment/roles/bro/tasks/bro.yml | 59 -
.../roles/bro/tasks/dependencies.yml | 48 -
.../roles/bro/tasks/librdkafka.yml | 39 -
metron-deployment/roles/bro/tasks/main.yml | 22 -
.../roles/bro/tasks/metron-bro-plugin-kafka.yml | 49 -
metron-deployment/roles/bro/tasks/nic.yml | 19 -
metron-deployment/roles/bro/vars/main.yml | 34 -
.../roles/build-tools/meta/main.yml | 19 -
.../roles/build-tools/tasks/main.yml | 34 -
.../roles/deployment-report/defaults/main.yml | 18 -
.../roles/deployment-report/meta/main.yml | 19 -
.../roles/deployment-report/tasks/main.yml | 37 -
metron-deployment/roles/epel/tasks/main.yml | 19 -
.../roles/fastcapa/defaults/main.yml | 42 -
metron-deployment/roles/fastcapa/meta/main.yml | 19 -
.../roles/fastcapa/tasks/debug.yml | 26 -
.../roles/fastcapa/tasks/dependencies.yml | 39 -
metron-deployment/roles/fastcapa/tasks/dpdk.yml | 59 -
.../roles/fastcapa/tasks/fastcapa.yml | 46 -
.../roles/fastcapa/tasks/kernel.yml | 51 -
metron-deployment/roles/fastcapa/tasks/main.yml | 22 -
.../roles/fastcapa/templates/fastcapa | 145 -
.../roles/fastcapa/templates/fastcapa.conf | 67 -
metron-deployment/roles/httplib2/tasks/main.yml | 20 -
.../roles/java_jdk/defaults/main.yml | 18 -
metron-deployment/roles/java_jdk/tasks/main.yml | 34 -
.../roles/kafka-broker/defaults/main.yml | 18 -
.../roles/kafka-broker/meta/main.yml | 18 -
.../roles/kafka-broker/tasks/main.yml | 41 -
.../roles/kafka-client/defaults/main.yml | 18 -
.../roles/kafka-client/tasks/main.yml | 30 -
.../roles/librdkafka/defaults/main.yml | 20 -
.../roles/librdkafka/tasks/dependencies.yml | 40 -
.../roles/librdkafka/tasks/librdkafka.yml | 39 -
.../roles/librdkafka/tasks/main.yml | 19 -
.../roles/libselinux-python/tasks/main.yml | 25 -
.../roles/load_web_templates/meta/main.yml | 19 -
.../roles/load_web_templates/tasks/main.yml | 24 -
.../roles/metron-builder/defaults/main.yml | 18 -
.../roles/metron-builder/tasks/main.yml | 26 -
.../roles/metron-rpms/defaults/main.yml | 18 -
.../roles/metron-rpms/tasks/copy_rpms.yml | 26 -
.../metron-rpms/tasks/create_directory.yml | 28 -
.../roles/metron-rpms/tasks/create_repo.yml | 28 -
.../roles/metron-rpms/tasks/main.yml | 21 -
.../files/es_templates/error_index.template | 57 -
.../roles/metron_pcapservice/defaults/main.yml | 29 -
.../roles/metron_pcapservice/meta/main.yml | 19 -
.../metron_pcapservice/tasks/config-hbase.yml | 26 -
.../roles/metron_pcapservice/tasks/main.yml | 25 -
.../metron_pcapservice/tasks/pcapservice.yml | 30 -
.../templates/metron-pcapservice-logrotate.yml | 27 -
.../metron_pcapservice/templates/pcapservice | 84 -
.../roles/metron_streaming/defaults/main.yml | 98 -
.../roles/metron_streaming/files/extractor.json | 12 -
.../roles/metron_streaming/meta/main.yml | 21 -
.../metron_streaming/tasks/copy_bundles.yml | 52 -
.../roles/metron_streaming/tasks/es_purge.yml | 42 -
.../roles/metron_streaming/tasks/geoip.yml | 22 -
.../metron_streaming/tasks/grok_upload.yml | 37 -
.../metron_streaming/tasks/hdfs_filesystem.yml | 56 -
.../roles/metron_streaming/tasks/hdfs_purge.yml | 52 -
.../roles/metron_streaming/tasks/main.yml | 53 -
.../metron_streaming/tasks/source_config.yml | 39 -
.../metron_streaming/tasks/threat_intel.yml | 46 -
.../roles/metron_streaming/tasks/topologies.yml | 86 -
.../templates/config/elasticsearch.global.json | 7 -
.../templates/config/solr.global.json | 6 -
.../metron_streaming/templates/threat_ip.csv | 37 -
.../roles/monit-start/defaults/main.yml | 23 -
.../roles/monit-start/tasks/main.yml | 53 -
metron-deployment/roles/monit/README.md | 79 -
metron-deployment/roles/monit/defaults/main.yml | 24 -
metron-deployment/roles/monit/tasks/main.yml | 28 -
.../roles/monit/tasks/monit-definitions.yml | 22 -
.../monit/tasks/monit-sensor-definitions.yml | 38 -
.../monit/tasks/monit-stub-definitions.yml | 30 -
metron-deployment/roles/monit/tasks/monit.yml | 27 -
.../roles/monit/templates/monit/bro-stub.monit | 25 -
.../roles/monit/templates/monit/bro.monit | 25 -
.../roles/monit/templates/monit/monit.conf | 29 -
.../monit/templates/monit/pcap-replay.monit | 24 -
.../roles/monit/templates/monit/pycapa.monit | 24 -
.../monit/templates/monit/snort-stub.monit | 25 -
.../roles/monit/templates/monit/snort.monit | 33 -
.../roles/monit/templates/monit/yaf-stub.monit | 25 -
.../roles/monit/templates/monit/yaf.monit | 24 -
metron-deployment/roles/ntp/tasks/main.yml | 31 -
metron-deployment/roles/opentaxii/README.md | 178 -
.../roles/opentaxii/defaults/main.yml | 43 -
metron-deployment/roles/opentaxii/meta/main.yml | 17 -
.../roles/opentaxii/tasks/dependencies.yml | 37 -
.../roles/opentaxii/tasks/hailataxii.yml | 45 -
.../roles/opentaxii/tasks/main.yml | 20 -
.../roles/opentaxii/tasks/opentaxii.yml | 50 -
.../opentaxii/templates/collection-status.py | 32 -
.../roles/opentaxii/templates/collections.yml | 19 -
.../roles/opentaxii/templates/opentaxii | 176 -
.../opentaxii/templates/opentaxii-conf.yml | 38 -
.../roles/opentaxii/templates/services.yml | 69 -
metron-deployment/roles/pcap_replay/README.md | 61 -
.../roles/pcap_replay/defaults/main.yml | 21 -
.../roles/pcap_replay/meta/main.yml | 17 -
.../roles/pcap_replay/tasks/dependencies.yml | 32 -
.../roles/pcap_replay/tasks/main.yml | 20 -
.../roles/pcap_replay/tasks/service.yml | 22 -
.../roles/pcap_replay/tasks/tcpreplay.yml | 38 -
.../roles/pcap_replay/templates/pcap-replay | 97 -
.../roles/pycapa/defaults/main.yml | 26 -
metron-deployment/roles/pycapa/meta/main.yml | 20 -
.../roles/pycapa/tasks/dependencies.yml | 36 -
metron-deployment/roles/pycapa/tasks/main.yml | 21 -
.../roles/pycapa/tasks/pycapa-service.yml | 22 -
metron-deployment/roles/pycapa/tasks/pycapa.yml | 40 -
metron-deployment/roles/pycapa/templates/pycapa | 115 -
.../roles/python-pip/tasks/main.yml | 25 -
metron-deployment/roles/sensor-stubs/README.md | 99 -
.../roles/sensor-stubs/defaults/main.yml | 25 -
.../roles/sensor-stubs/files/bro.out | 1346 -
.../roles/sensor-stubs/files/snort.out | 27404 -----------------
.../roles/sensor-stubs/files/yaf.out | 22164 -------------
.../roles/sensor-stubs/tasks/main.yml | 40 -
.../roles/sensor-stubs/templates/sensor-stubs | 154 -
.../roles/sensor-stubs/templates/start-bro-stub | 55 -
.../sensor-stubs/templates/start-snort-stub | 55 -
.../roles/sensor-stubs/templates/start-yaf-stub | 56 -
.../roles/sensor-test-mode/README.md | 44 -
.../roles/sensor-test-mode/defaults/main.yml | 20 -
.../roles/sensor-test-mode/files/example.pcap | Bin 507865 -> 0 bytes
.../roles/sensor-test-mode/meta/main.yml | 19 -
.../roles/sensor-test-mode/tasks/main.yml | 25 -
.../roles/sensor-test-mode/tasks/pcap.yml | 22 -
.../roles/sensor-test-mode/tasks/snort.yml | 36 -
.../roles/sensor-test-mode/tasks/yaf.yml | 30 -
metron-deployment/roles/snort/defaults/main.yml | 28 -
metron-deployment/roles/snort/files/snort.conf | 730 -
metron-deployment/roles/snort/meta/main.yml | 23 -
metron-deployment/roles/snort/tasks/daq.yml | 36 -
metron-deployment/roles/snort/tasks/main.yml | 21 -
metron-deployment/roles/snort/tasks/nic.yml | 19 -
.../roles/snort/tasks/producer.yml | 28 -
metron-deployment/roles/snort/tasks/snort.yml | 85 -
.../roles/snort/templates/snort-producer | 83 -
.../snort/templates/start-snort-producer.sh | 24 -
.../roles/tap_interface/defaults/main.yml | 19 -
.../roles/tap_interface/tasks/main.yml | 35 -
metron-deployment/roles/yaf/defaults/main.yml | 29 -
metron-deployment/roles/yaf/meta/main.yml | 22 -
metron-deployment/roles/yaf/tasks/fixbuf.yml | 37 -
metron-deployment/roles/yaf/tasks/main.yml | 20 -
metron-deployment/roles/yaf/tasks/nic.yml | 19 -
metron-deployment/roles/yaf/tasks/yaf.yml | 50 -
.../roles/yaf/templates/start-yaf.sh | 25 -
metron-deployment/roles/yaf/templates/yaf | 83 -
metron-deployment/vagrant/README.md | 21 -
.../vagrant/fastcapa-test-platform/.gitignore | 1 -
.../vagrant/fastcapa-test-platform/README.md | 138 -
.../centos-7.1/Vagrantfile | 72 -
.../centos-7.1/ansible.cfg | 22 -
.../centos-7.1/vars/main.yml | 52 -
.../centos-7.4/Vagrantfile | 72 -
.../centos-7.4/ansible.cfg | 22 -
.../centos-7.4/vars/main.yml | 52 -
.../vagrant/fastcapa-test-platform/playbook.yml | 49 -
.../tasks/validate-messages-received.yml | 38 -
.../tasks/validate-packets-sent.yml | 41 -
.../vagrant/full-dev-platform/.gitignore | 1 -
.../vagrant/full-dev-platform/README.md | 123 -
.../vagrant/full-dev-platform/Vagrantfile | 92 -
.../vagrant/full-dev-platform/ansible.cfg | 27 -
.../full-dev-platform/run_ansible_role.sh | 20 -
.../full-dev-platform/run_enrichment_role.sh | 20 -
metron-interface/metron-rest/README.md | 6 +-
436 files changed, 61195 insertions(+), 60980 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
index 86d61e7..4858807 100644
--- a/README.md
+++ b/README.md
@@ -16,7 +16,7 @@ See the License for the specific language governing permissions and
limitations under the License.
-->
[](https://travis-ci.org/apache/metron)
-
+
# Apache Metron
Metron integrates a variety of open source big data technologies in order
@@ -27,36 +27,36 @@ the most current threat intelligence information to security telemetry
within a single platform.
For the latest information, please visit our website at http://metron.apache.org/
-
+
Metron can be divided into 4 areas:
1. **A mechanism to capture, store, and normalize any type of security
telemetry at extremely high rates.** Because security telemetry is constantly
-being generated, it requires a method for ingesting the data at high speeds
+being generated, it requires a method for ingesting the data at high speeds
and pushing it to various processing units for advanced computation and analytics.
2. **Real time processing and application of enrichments** such as threat
intelligence, geolocation, and DNS information to telemetry being collected.
The immediate application of this information to incoming telemetry provides
-the context and situational awareness, as well as the who and where
+the context and situational awareness, as well as the who and where
information critical for investigation
3. **Efficient information storage** based on how the information will be used:
- Logs and telemetry are stored such that they can be efficiently mined and
analyzed for concise security visibility
- - The ability to extract and reconstruct full packets helps an analyst answer
-questions such as who the true attacker was, what data was leaked, and where
+ - The ability to extract and reconstruct full packets helps an analyst answer
+questions such as who the true attacker was, what data was leaked, and where
that data was sent
- - Long-term storage not only increases visibility over time, but also enables
-advanced analytics such as machine learning techniques to be used to create
-models on the information. Incoming data can then be scored against these
+ - Long-term storage not only increases visibility over time, but also enables
+advanced analytics such as machine learning techniques to be used to create
+models on the information. Incoming data can then be scored against these
stored models for advanced anomaly detection.
-4. **An interface that gives a security investigator a centralized view of data
-and alerts passed through the system.** Metron’s interface presents alert
-summaries with threat intelligence and enrichment data specific to that alert
-on one single page. Furthermore, advanced search capabilities and full packet
-extraction tools are presented to the analyst for investigation without the
+4. **An interface that gives a security investigator a centralized view of data
+and alerts passed through the system.** Metron’s interface presents alert
+summaries with threat intelligence and enrichment data specific to that alert
+on one single page. Furthermore, advanced search capabilities and full packet
+extraction tools are presented to the analyst for investigation without the
need to pivot into additional tools.
Big data is a natural fit for powerful security analytics. The Metron
@@ -83,7 +83,7 @@ Option 3 is more likely to have the latest code.
# Getting Started
-To start exploring the capabilities of Apache Metron [follow these instructions to launch Metron in a single-node VM](metron-deployment/vagrant/full-dev-platform) on your own hardware.
+To start exploring the capabilities of Apache Metron [follow these instructions to launch Metron in a single-node VM](metron-deployment/development/centos6) on your own hardware.
# Building Metron
@@ -146,4 +146,4 @@ Some useful utilities that cross all of these parts of the architecture:
In order to allow for meta alerts to be queries alongside regular alerts in Elasticsearch 2.x,
it is necessary to add an additional field to the templates and mapping for existing sensors.
-Please see a description of the steps necessary to make this change in the metron-elasticsearch [Using Metron with Elasticsearch 2.x](./metron-platform/metron-elasticsearch#using-metron-with-elasticsearch-2x)
\ No newline at end of file
+Please see a description of the steps necessary to make this change in the metron-elasticsearch [Using Metron with Elasticsearch 2.x](./metron-platform/metron-elasticsearch#using-metron-with-elasticsearch-2x)
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/dev-utilities/release-utils/metron-rc-check
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/metron-rc-check b/dev-utilities/release-utils/metron-rc-check
index da5412e..142cd39 100755
--- a/dev-utilities/release-utils/metron-rc-check
+++ b/dev-utilities/release-utils/metron-rc-check
@@ -260,7 +260,7 @@ echo ""
read -p " run vagrant full_dev? [yN] " -n 1 -r
echo
if [[ $REPLY =~ ^[Yy]$ ]]; then
- cd "$WORK/apache-metron-$METRON_VERSION-$RC/metron-deployment/vagrant/full-dev-platform" || exit 1
+ cd "$WORK/apache-metron-$METRON_VERSION-$RC/metron-deployment/development/centos6" || exit 1
if [[ ${DID_BUILD} -ne 1 ]]; then
vagrant up
else
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-contrib/metron-docker/README.md
----------------------------------------------------------------------
diff --git a/metron-contrib/metron-docker/README.md b/metron-contrib/metron-docker/README.md
index c290732..35364d4 100644
--- a/metron-contrib/metron-docker/README.md
+++ b/metron-contrib/metron-docker/README.md
@@ -19,12 +19,12 @@ limitations under the License.
Metron Docker is a [Docker Compose](https://docs.docker.com/compose/overview/) application that is intended only for development and integration testing of Metron. These images can quickly spin-up the underlying components on which Apache Metron runs.
-None of the core Metron components are setup or launched automatically with these Docker images. You will need to manually setup and start the Metron components that you require. You should not expect to see telemetry being parsed, enriched, or indexed. If you are looking to try-out, experiment or demo Metron capabilities on a single node, then the [Vagrant-driven VM](../../metron-deployment/vagrant/full-dev-platform) is what you need. Use this instead of Vagrant when:
-
+None of the core Metron components are setup or launched automatically with these Docker images. You will need to manually setup and start the Metron components that you require. You should not expect to see telemetry being parsed, enriched, or indexed. If you are looking to try-out, experiment or demo Metron capabilities on a single node, then the [Vagrant-driven VM](../../metron-deployment/development/centos6) is what you need. Use this instead of Vagrant when:
+
- You want an environment that can be built and spun up quickly
- You need to frequently rebuild and restart services
- You only need to test, troubleshoot or develop against a subset of services
-
+
Metron Docker includes these images that have been customized for Metron:
- Kafka (with Zookeeper)
@@ -42,7 +42,7 @@ Install [Docker for Mac](https://docs.docker.com/docker-for-mac/) or [Docker for
- Docker version 1.12.0
- docker-machine version 0.8.0
- docker-compose version 1.8.0
-
+
Build Metron from the top level directory with:
```
$ cd $METRON_HOME
@@ -52,7 +52,7 @@ $ mvn clean install -DskipTests
You are welcome to use an existing Docker host but we prefer one with more resources. You can create one of those with this script:
```
$ export METRON_DOCKER_HOME=$METRON_HOME/metron-contrib/metron-docker
-$ cd $METRON_DOCKER_HOME
+$ cd $METRON_DOCKER_HOME
$ ./scripts/create-docker-machine.sh
```
@@ -161,7 +161,7 @@ $ docker-compose exec kafkazk ./bin/produce-data.sh
Usage: produce-data.sh data_path topic [message_delay_in_seconds]
# Stream data in TestData.txt to the 'test' Kafka topic at a frequency of 5 seconds (default is 1 second)
-$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5
+$ docker-compose exec kafkazk ./bin/produce-data.sh /data/TestData.txt test 5
```
The Kafka/Zookeeper image comes with sample Bro and Squid data:
@@ -269,4 +269,4 @@ $ curl -XGET http://192.168.99.100:9200/_cat/indices?v
health status index pri rep docs.count docs.deleted store.size pri.store.size
yellow open .kibana 1 1 1 0 3.1kb 3.1kb
yellow open bro_index_2016.12.19.18 5 1 180 0 475kb 475kb
-```
\ No newline at end of file
+```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/Kerberos-manual-setup.md
----------------------------------------------------------------------
diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md
index 5d17cf0..47a63d8 100644
--- a/metron-deployment/Kerberos-manual-setup.md
+++ b/metron-deployment/Kerberos-manual-setup.md
@@ -34,7 +34,7 @@ This document provides instructions for kerberizing Metron's Vagrant-based devel
Setup
-----
-1. Deploy the [development environment.](vagrant/full-dev-platform/README.md).
+1. Deploy the [development environment.](development/centos6/README.md).
1. Export the following environment variables. These need to be set for the remainder of the instructions. Replace `node1` with the appropriate hosts, if you are running Metron anywhere other than Vagrant.
@@ -90,7 +90,7 @@ Setup a KDC
cp -f /etc/krb5.conf /var/lib/ambari-server/resources/scripts
```
-1. Ensure that the KDC can issue renewable tickets. This may be necessary on a real cluster, but should not be on [Full Dev](vagrant/full-dev-platform/README.md).
+1. Ensure that the KDC can issue renewable tickets. This may be necessary on a real cluster, but should not be on a [single VM](development/centos6/README.md).
Edit `/var/kerberos/krb5kdc/kdc.conf` and ensure the following is added to the `realm` section
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/README.md b/metron-deployment/README.md
index 86d04e7..805aaf0 100644
--- a/metron-deployment/README.md
+++ b/metron-deployment/README.md
@@ -67,7 +67,7 @@ Running Metron within the resource constraints of a single VM is incredibly chal
#### How?
-To deploy Metron in a VM running on your computer, follow the instructions at [vagrant/full-dev-platform](vagrant/full-dev-platform)
+To deploy Metron in a VM running on your computer, follow the instructions at [development/centos6](development/centos6).
How do I build RPM packages?
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/amazon-ec2/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/ansible.cfg b/metron-deployment/amazon-ec2/ansible.cfg
index 9f3a2fe..1f4f0ea 100644
--- a/metron-deployment/amazon-ec2/ansible.cfg
+++ b/metron-deployment/amazon-ec2/ansible.cfg
@@ -16,8 +16,8 @@
#
[defaults]
host_key_checking = False
-library = ../extra_modules
-roles_path = ../roles
+library = ../ansible/extra_modules
+roles_path = ../ansible/roles
pipelining = True
remote_user = centos
forks = 20
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index 124d7ab..b4e24f8 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -15,14 +15,6 @@
# limitations under the License.
#
---
-# which services should be started?
-services_to_start:
- - yaf
- - snort
- - snort-logs
- - bro
- - pcap-replay
-
# the ami for cent6 by region
amis_by_region:
us-east-1: ami-1c221e76 # US East (N. Virginia)
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/playbook.yml b/metron-deployment/amazon-ec2/playbook.yml
index de51534..de64490 100644
--- a/metron-deployment/amazon-ec2/playbook.yml
+++ b/metron-deployment/amazon-ec2/playbook.yml
@@ -71,4 +71,4 @@
#
# build the metron cluster
#
-- include: ../playbooks/metron_full_install.yml
+- include: ../ansible/playbooks/metron_full_install.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/README.md b/metron-deployment/ansible/README.md
new file mode 100644
index 0000000..d35e755
--- /dev/null
+++ b/metron-deployment/ansible/README.md
@@ -0,0 +1,24 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Ansible Assets
+
+This directory contains all of the shared Ansible assets used to deploy Metron in a [development environment](../development/README.md). The scope of what Ansible is used for is intentionally limited. The recommended means of deploying Metron is to use the [Metron MPack](../packaging/ambari/metron-mpack/README.md) for [Apache Ambari](https://ambari.apache.org/).
+
+Ansible is only used primarily to prepare the development environment for Ambari and for deploying a suite of test sensors to drive telemetry through Metron. The Ansible assets contained here are of limited use outside of this scenario.
+
+**Warning** It is not recommended that you use these assets for deploying Metron in your environment for either production or testing. Support for this use case cannot be provided.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/extra_modules/ambari_cluster_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/extra_modules/ambari_cluster_state.py b/metron-deployment/ansible/extra_modules/ambari_cluster_state.py
new file mode 100644
index 0000000..0542526
--- /dev/null
+++ b/metron-deployment/ansible/extra_modules/ambari_cluster_state.py
@@ -0,0 +1,395 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DOCUMENTATION = '''
+---
+module: ambari_cluster_state
+version_added: "2.1"
+author: Mark Bittmann (https://github.com/mbittmann)
+short_description: Create, delete, start or stop an ambari cluster
+description:
+ - Create, delete, start or stop an ambari cluster
+options:
+ host:
+ description:
+ The hostname for the ambari web server
+ port:
+ description:
+ The port for the ambari web server
+ username:
+ description:
+ The username for the ambari web server
+ password:
+ description:
+ The name of the cluster in web server
+ required: yes
+ cluster_name:
+ description:
+ The name of the cluster in ambari
+ required: yes
+ cluster_state:
+ description:
+ The desired state for the ambari cluster ['present', 'absent', 'started', 'stopped']. Setting the cluster
+ state to absent will first stop the cluster.
+ required: yes
+ blueprint_var:
+ description:
+ The path to the file defining the cluster blueprint and host mapping. Required when state == 'present'
+ required: no
+ blueprint_name:
+ description:
+ The name of the blueprint. Required when state == 'present'
+ required: no
+ wait_for_complete:
+ description:
+ Whether to wait for the request to complete before returning. Default is False.
+ required: no
+ requirements: [ 'requests']
+'''
+
+EXAMPLES = '''
+# must use full relative path to any files in stored in roles/role_name/files/
+- name: Create a new ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: present
+ blueprint_var: roles/my_role/files/blueprint.yml
+ blueprint_name: hadoop
+ wait_for_complete: True
+- name: Start the ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: started
+ wait_for_complete: True
+- name: Stop the ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: stopped
+ wait_for_complete: True
+- name: Delete the ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: absent
+'''
+
+RETURN = '''
+results:
+ description: The content of the requests object returned from the RESTful call
+ returned: success
+ type: string
+created_blueprint:
+ description: Whether a blueprint was created
+ returned: success
+ type: boolean
+status:
+ description: The status of the blueprint creation process
+ returned: success
+ type: string
+'''
+
+__author__ = 'mbittmann'
+
+import json
+try:
+ import requests
+except ImportError:
+ REQUESTS_FOUND = False
+else:
+ REQUESTS_FOUND = True
+
+
+def main():
+
+ argument_spec = dict(
+ host=dict(type='str', default=None, required=True),
+ port=dict(type='int', default=None, required=True),
+ username=dict(type='str', default=None, required=True),
+ password=dict(type='str', default=None, required=True),
+ cluster_name=dict(type='str', default=None, required=True),
+ cluster_state=dict(type='str', default=None, required=True,
+ choices=['present', 'absent', 'started', 'stopped']),
+ blueprint_var=dict(type='dict', required=False),
+ blueprint_name=dict(type='str', default=None, required=False),
+ configurations=dict(type='list', default=None, required=False),
+ wait_for_complete=dict(default=False, required=False, type='bool'),
+ )
+
+ required_together = ['blueprint_var', 'blueprint_name']
+
+ module = AnsibleModule(
+ argument_spec=argument_spec,
+ required_together=required_together
+ )
+
+ if not REQUESTS_FOUND:
+ module.fail_json(
+ msg='requests library is required for this module')
+
+ p = module.params
+
+ host = p.get('host')
+ port = p.get('port')
+ username = p.get('password')
+ password = p.get('password')
+ cluster_name = p.get('cluster_name')
+ cluster_state = p.get('cluster_state')
+ blueprint_name = p.get('blueprint_name')
+ wait_for_complete = p.get('wait_for_complete')
+
+ ambari_url = 'http://{0}:{1}'.format(host, port)
+
+ try:
+ if cluster_state in ['started', 'stopped']:
+ if not cluster_exists(ambari_url, username, password, cluster_name):
+ module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name))
+ state = ''
+ if cluster_state == 'started':
+ state = 'STARTED'
+ elif cluster_state == 'stopped':
+ state = 'INSTALLED'
+
+ request = set_cluster_state(ambari_url, username, password, cluster_name, state)
+ if wait_for_complete:
+ try:
+ request_id = json.loads(request.content)['Requests']['id']
+ except ValueError:
+ module.exit_json(changed=True, results=request.content)
+ status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
+ if status != 'COMPLETED':
+ module.fail_json(msg="Request failed with status {0}".format(status))
+ module.exit_json(changed=True, results=request.content)
+ elif cluster_state == 'absent':
+ if not cluster_exists(ambari_url, username, password, cluster_name):
+ module.exit_json(changed=False, msg='Skipping. Cluster does not exist')
+ if not can_delete_cluster(ambari_url, username, password, cluster_name):
+ request = set_cluster_state(ambari_url, username, password, cluster_name, 'INSTALLED')
+ request_id = json.loads(request.content)['Requests']['id']
+ status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
+ if status != 'COMPLETED':
+ module.fail_json(msg="Request failed with status {0}".format(status))
+ request = delete_cluster(ambari_url, username, password, cluster_name)
+ module.exit_json(changed=True, results=request.content)
+ elif cluster_state == 'present':
+ if not p.get('blueprint_var') or not blueprint_name: # have neither name nor file
+ module.fail_json(msg="Must provide blueprint_var and blueprint_name when cluster_state=='present'")
+
+ blueprint_var = p.get('blueprint_var')
+ blueprint, host_map = blueprint_var_to_ambari_converter(blueprint_var)
+ created_blueprint = False
+
+ if not blueprint_exists(ambari_url, username, password, blueprint_name):
+ create_blueprint(ambari_url, username, password, blueprint_name, blueprint)
+ created_blueprint = True
+
+ if cluster_exists(ambari_url, username, password, cluster_name):
+ module.exit_json(changed=False, msg='Cluster {0} already exists'.format(cluster_name),
+ created_blueprint=created_blueprint)
+
+ configurations = p.get('configurations')
+ request = create_cluster(ambari_url, username, password, cluster_name, blueprint_name, configurations, host_map)
+ request_id = json.loads(request.content)['Requests']['id']
+ if wait_for_complete:
+ status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
+ if status != 'COMPLETED':
+ module.fail_json(msg="Request failed with status {0}".format(status))
+ request_status = get_request_status(ambari_url, username, password, cluster_name, request_id)
+ module.exit_json(changed=True, results=request.content,
+ created_blueprint=created_blueprint, status=request_status)
+
+ except requests.ConnectionError, e:
+ module.fail_json(msg="Could not connect to Ambari client: " + str(e.message))
+ except Exception, e:
+ module.fail_json(msg="Ambari client exception occurred: " + str(e.message))
+
+
+def get_clusters(ambari_url, user, password):
+ r = get(ambari_url, user, password, '/api/v1/clusters')
+ if r.status_code != 200:
+ msg = 'Could not get cluster list: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ clusters = json.loads(r.content)
+ return clusters['items']
+
+
+def cluster_exists(ambari_url, user, password, cluster_name):
+ clusters = get_clusters(ambari_url, user, password)
+ return cluster_name in [item['Clusters']['cluster_name'] for item in clusters]
+
+
+def set_cluster_state(ambari_url, user, password, cluster_name, cluster_state):
+ path = '/api/v1/clusters/{0}/services'.format(cluster_name)
+ request = {"RequestInfo": {"context": "Setting cluster state"},
+ "Body": {"ServiceInfo": {"state": "{0}".format(cluster_state)}}}
+ payload = json.dumps(request)
+ r = put(ambari_url, user, password, path, payload)
+ if r.status_code not in [202, 200]:
+ msg = 'Could not set cluster state: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def create_cluster(ambari_url, user, password, cluster_name, blueprint_name, configurations, hosts_json):
+ path = '/api/v1/clusters/{0}'.format(cluster_name)
+ data = json.dumps({'blueprint': blueprint_name, 'configurations': configurations, 'host_groups': hosts_json})
+ f = open('cluster.log', 'w')
+ f.write(data)
+ f.close()
+ r = post(ambari_url, user, password, path, data)
+ if r.status_code != 202:
+ msg = 'Could not create cluster: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def get_request_status(ambari_url, user, password, cluster_name, request_id):
+ path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id)
+ r = get(ambari_url, user, password, path)
+ if r.status_code != 200:
+ msg = 'Could not get cluster request status: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ service = json.loads(r.content)
+ return service['Requests']['request_status']
+
+
+def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time):
+ while True:
+ status = get_request_status(ambari_url, user, password, cluster_name, request_id)
+ if status == 'COMPLETED':
+ return status
+ elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']:
+ return status
+ else:
+ time.sleep(sleep_time)
+
+
+def can_delete_cluster(ambari_url, user, password, cluster_name):
+ path = '/api/v1/clusters/{0}/services?ServiceInfo/state=STARTED'.format(cluster_name)
+ r = get(ambari_url, user, password, path)
+ items = json.loads(r.content)['items']
+ return len(items) > 0
+
+
+def get_blueprints(ambari_url, user, password):
+ path = '/api/v1/blueprints'
+ r = get(ambari_url, user, password, path)
+ if r.status_code != 200:
+ msg = 'Could not get blueprint list: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+
+ services = json.loads(r.content)
+ return services['items']
+
+
+def create_blueprint(ambari_url, user, password, blueprint_name, blueprint_data):
+ data = json.dumps(blueprint_data)
+ f = open('blueprint.log', 'w')
+ f.write(data)
+ f.close()
+ path = "/api/v1/blueprints/" + blueprint_name
+ r = post(ambari_url, user, password, path, data)
+ if r.status_code != 201:
+ msg = 'Could not create blueprint: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def blueprint_exists(ambari_url, user, password, blueprint_name):
+ blueprints = get_blueprints(ambari_url, user, password)
+ return blueprint_name in [item['Blueprints']['blueprint_name'] for item in blueprints]
+
+
+def delete_cluster(ambari_url, user, password, cluster_name):
+ path = '/api/v1/clusters/{0}'.format(cluster_name)
+ r = delete(ambari_url, user, password, path)
+ if r.status_code != 200:
+ msg = 'Could not delete cluster: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def get(ambari_url, user, password, path):
+ r = requests.get(ambari_url + path, auth=(user, password))
+ return r
+
+
+def put(ambari_url, user, password, path, data):
+ headers = {'X-Requested-By': 'ambari'}
+ r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers)
+ return r
+
+
+def post(ambari_url, user, password, path, data):
+ headers = {'X-Requested-By': 'ambari'}
+ r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers)
+ return r
+
+
+def delete(ambari_url, user, password, path):
+ headers = {'X-Requested-By': 'ambari'}
+ r = requests.delete(ambari_url + path, auth=(user, password), headers=headers)
+ return r
+
+
+def blueprint_var_to_ambari_converter(blueprint_var):
+ groups = blueprint_var['groups']
+ new_groups = []
+ host_map = []
+ for group in groups:
+ components = []
+ for component in group['components']:
+ components.append({'name': component})
+ group['components'] = components
+ hosts = group.pop('hosts')
+ new_groups.append(group)
+ this_host_map = dict()
+ this_host_map['name'] = group['name']
+ this_host_list = [{'fqdn': host} for host in hosts]
+ this_host_map['hosts'] = this_host_list
+ host_map.append(this_host_map)
+ blueprint = dict()
+ blueprint['configurations'] = blueprint_var['required_configurations']
+ blueprint['host_groups'] = new_groups
+ blueprint['Blueprints'] = {'stack_name': blueprint_var['stack_name'], 'stack_version': blueprint_var['stack_version']}
+ return blueprint, host_map
+
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/extra_modules/ambari_service_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/extra_modules/ambari_service_state.py b/metron-deployment/ansible/extra_modules/ambari_service_state.py
new file mode 100644
index 0000000..6d44e8d
--- /dev/null
+++ b/metron-deployment/ansible/extra_modules/ambari_service_state.py
@@ -0,0 +1,352 @@
+#!/usr/bin/python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DOCUMENTATION = '''
+---
+module: ambari_service_state
+version_added: "2.1"
+author: Apache Metron (https://metron.apache.org)
+short_description: Start/Stop/Change Service or Component State
+description:
+ - Start/Stop/Change Service or Component State
+options:
+ host:
+ description:
+ The hostname for the ambari web server
+ port:
+ description:
+ The port for the ambari web server
+ username:
+ description:
+ The username for the ambari web server
+ password:
+ description:
+ The name of the cluster in web server
+ required: yes
+ cluster_name:
+ description:
+ The name of the cluster in ambari
+ required: yes
+ service_name:
+ description:
+ The name of the service to alter
+ required: no
+ component_name:
+ description:
+ The name of the component to alter
+ required: no
+ component_host:
+ description:
+ The host running the targeted component. Required when component_name is used.
+ required: no
+ state:
+ description:
+ The desired service/component state.
+ wait_for_complete:
+ description:
+ Whether to wait for the request to complete before returning. Default is False.
+ required: no
+ requirements: [ 'requests']
+'''
+
+EXAMPLES = '''
+# must use full relative path to any files in stored in roles/role_name/files/
+- name: Create a new ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: present
+ blueprint_var: roles/my_role/files/blueprint.yml
+ blueprint_name: hadoop
+ wait_for_complete: True
+- name: Start the ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: started
+ wait_for_complete: True
+- name: Stop the ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: stopped
+ wait_for_complete: True
+- name: Delete the ambari cluster
+ ambari_cluster_state:
+ host: localhost
+ port: 8080
+ username: admin
+ password: admin
+ cluster_name: my_cluster
+ cluster_state: absent
+'''
+
+RETURN = '''
+results:
+ description: The content of the requests object returned from the RESTful call
+ returned: success
+ type: string
+'''
+
+__author__ = 'apachemetron'
+
+import json
+
+try:
+ import requests
+except ImportError:
+ REQUESTS_FOUND = False
+else:
+ REQUESTS_FOUND = True
+
+
+def main():
+
+ argument_spec = dict(
+ host=dict(type='str', default=None, required=True),
+ port=dict(type='int', default=None, required=True),
+ username=dict(type='str', default=None, required=True),
+ password=dict(type='str', default=None, required=True),
+ cluster_name=dict(type='str', default=None, required=True),
+ state=dict(type='str', default=None, required=True,
+ choices=['started', 'stopped', 'deleted']),
+ service_name=dict(type='str', required=False),
+ component_name=dict(type='str', default=None, required=False),
+ component_host=dict(type='str', default=None, required=False),
+ wait_for_complete=dict(default=False, required=False, type='bool'),
+ )
+
+ required_together = ['component_name', 'component_host']
+
+ module = AnsibleModule(
+ argument_spec=argument_spec,
+ required_together=required_together
+ )
+
+ if not REQUESTS_FOUND:
+ module.fail_json(
+ msg='requests library is required for this module')
+
+ p = module.params
+
+ host = p.get('host')
+ port = p.get('port')
+ username = p.get('username')
+ password = p.get('password')
+ cluster_name = p.get('cluster_name')
+ state = p.get('state')
+ service_name = p.get('service_name')
+ component_name = p.get('component_name')
+ component_host = p.get('component_host')
+ wait_for_complete = p.get('wait_for_complete')
+ component_mode = False
+ ambari_url = 'http://{0}:{1}'.format(host, port)
+
+ if component_name:
+ component_mode = True
+
+ try:
+ if not cluster_exists(ambari_url, username, password, cluster_name):
+ module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name))
+
+ if state in ['started', 'stopped', 'installed']:
+ desired_state = ''
+
+ if state == 'started':
+ desired_state = 'STARTED'
+ elif state in ['stopped','installed']:
+ desired_state = 'INSTALLED'
+
+ if component_mode:
+ if desired_state == 'INSTALLED':
+ if(can_add_component(ambari_url, username, password, cluster_name, component_name, component_host)):
+ add_component_to_host(ambari_url, username, password, cluster_name, component_name, component_host)
+ request = set_component_state(ambari_url, username, password, cluster_name, component_name, component_host, desired_state)
+ else:
+ request = set_service_state(ambari_url,username,password,cluster_name,service_name, desired_state)
+ if wait_for_complete:
+ try:
+ request_id = json.loads(request.content)['Requests']['id']
+ except ValueError:
+ module.exit_json(changed=True, results=request.content)
+ status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
+ if status != 'COMPLETED':
+ module.fail_json(msg="Request failed with status {0}".format(status))
+ module.exit_json(changed=True, results=request.content)
+
+ elif state == 'deleted':
+ if component_mode:
+ request = delete_component(ambari_url, username, password, cluster_name, component_name, component_host)
+ else:
+ request = delete_service(ambari_url,username,password,cluster_name,service_name)
+ module.exit_json(changed=True, results=request.content)
+
+ except requests.ConnectionError, e:
+ module.fail_json(msg="Could not connect to Ambari client: " + str(e.message))
+ except Exception, e:
+ module.fail_json(msg="Ambari client exception occurred: " + str(e.message))
+
+
+def get_clusters(ambari_url, user, password):
+ r = get(ambari_url, user, password, '/api/v1/clusters')
+ if r.status_code != 200:
+ msg = 'Could not get cluster list: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ clusters = json.loads(r.content)
+ return clusters['items']
+
+
+def cluster_exists(ambari_url, user, password, cluster_name):
+ clusters = get_clusters(ambari_url, user, password)
+ return cluster_name in [item['Clusters']['cluster_name'] for item in clusters]
+
+
+def get_request_status(ambari_url, user, password, cluster_name, request_id):
+ path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id)
+ r = get(ambari_url, user, password, path)
+ if r.status_code != 200:
+ msg = 'Could not get cluster request status: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ service = json.loads(r.content)
+ return service['Requests']['request_status']
+
+
+def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time):
+ while True:
+ status = get_request_status(ambari_url, user, password, cluster_name, request_id)
+ if status == 'COMPLETED':
+ return status
+ elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']:
+ return status
+ else:
+ time.sleep(sleep_time)
+
+
+def set_service_state(ambari_url, user, password, cluster_name, service_name, desired_state):
+ path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name)
+ request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(service_name,desired_state)},
+ "Body": {"ServiceInfo": {"state": "{0}".format(desired_state)}}}
+ payload = json.dumps(request)
+ r = put(ambari_url, user, password, path, payload)
+ if r.status_code not in [202, 200]:
+ msg = 'Could not set service state: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def set_component_state(ambari_url, user, password, cluster_name, component_name, component_host, desired_state):
+ path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+ request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(component_name,desired_state)},
+ "Body": {"HostRoles": {"state": "{0}".format(desired_state)}}}
+ payload = json.dumps(request)
+ r = put(ambari_url, user, password, path, payload)
+ if r.status_code not in [202, 200]:
+ msg = 'Could not set component state: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def delete_component(ambari_url, user, password, cluster_name, component_name, component_host):
+ enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host)
+ path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+ r = delete(ambari_url,user,password,path)
+ if r.status_code not in [202, 200]:
+ msg = 'Could not set service state: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host):
+ path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+ request = {"RequestInfo":{"context":"Turn On Maintenance Mode for {0}".format(component_name)},
+ "Body":{"HostRoles":{"maintenance_state":"ON"}}}
+ payload = json.dumps(request)
+ r = put(ambari_url, user, password, path, payload)
+ if r.status_code not in [202, 200]:
+ msg = 'Could not set maintenance mode: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def delete_service(ambari_url, user, password, cluster_name, service_name):
+ path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name)
+ r = delete(ambari_url,user,password,path)
+ if r.status_code not in [202, 200]:
+ msg = 'Could not delete service: request code {0}, \
+ request message {1}'.format(r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def add_component_to_host(ambari_url, user, password, cluster_name, component_name, component_host):
+ path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+ r = post(ambari_url, user, password, path,'')
+ if r.status_code not in [202,201,200]:
+ msg = 'Could not add {0} to host {1}: request code {2}, \
+ request message {3}'.format(component_name,component_host,r.status_code, r.content)
+ raise Exception(msg)
+ return r
+
+
+def can_add_component(ambari_url, user, password, cluster_name, component_name, component_host):
+ path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
+ r = get(ambari_url, user, password, path)
+ return r.status_code == 404
+
+
+def get(ambari_url, user, password, path):
+ r = requests.get(ambari_url + path, auth=(user, password))
+ return r
+
+
+def put(ambari_url, user, password, path, data):
+ headers = {'X-Requested-By': 'ambari'}
+ r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers)
+ return r
+
+
+def post(ambari_url, user, password, path, data):
+ headers = {'X-Requested-By': 'ambari'}
+ r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers)
+ return r
+
+
+def delete(ambari_url, user, password, path):
+ headers = {'X-Requested-By': 'ambari'}
+ r = requests.delete(ambari_url + path, auth=(user, password), headers=headers)
+ return r
+
+
+from ansible.module_utils.basic import *
+if __name__ == '__main__':
+ main()
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/ambari_install.yml b/metron-deployment/ansible/playbooks/ambari_install.yml
new file mode 100644
index 0000000..5b802b3
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/ambari_install.yml
@@ -0,0 +1,54 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: ec2
+ become: true
+ tasks:
+ - include_vars: ../../amazon-ec2/conf/defaults.yml
+ tags:
+ - ec2
+
+- hosts: packer
+ become: true
+ tasks:
+ - include_vars: ../../development/centos6/ansible/inventory/group_vars/all
+ tags:
+ - packer
+
+- hosts: ambari_*
+ become: true
+ roles:
+ - role: ambari_common
+ tags:
+ - ambari-prereqs
+ - hdp-install
+
+- hosts: ambari_master
+ become: true
+ roles:
+ - role: ambari_master
+ tags:
+ - ambari-server
+ - hdp-install
+
+- hosts: ambari_slave
+ become: true
+ roles:
+ - role: ambari_slave
+ tags:
+ - ambari-agent
+ - hdp-install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/docker_probe_install.yml b/metron-deployment/ansible/playbooks/docker_probe_install.yml
new file mode 100644
index 0000000..7be779a
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/docker_probe_install.yml
@@ -0,0 +1,62 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# sensors
+#
+- hosts: localhost
+ tasks:
+ - name: add container to inventory
+ add_host:
+ name: amb-server
+ ansible_connection: docker
+ groups: sensors
+ changed_when: false
+ tags: add-host
+
+- hosts: sensors
+ vars:
+ metron_version: 0.4.3
+ metron_directory: /usr/metron/{{ metron_version }}
+ bro_version: "2.5.2"
+ fixbuf_version: "1.7.1"
+ yaf_version: "2.8.0"
+ daq_version: "2.0.6-1"
+ pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+ pycapa_home: "/opt/pycapa"
+ snort_version: "2.9.8.0-1"
+ snort_alert_csv_path: "/var/log/snort/alert.csv"
+ threat_intel_bulk_load: False
+ sensor_test_mode: True
+ install_pycapa: False
+ install_bro: True
+ install_snort: True
+ install_yaf: True
+ install_pcap_replay: True
+ sniff_interface: eth0
+ pcap_replay_interface: "{{ sniff_interface }}"
+ pcapservice_port: 8081
+ kafka_broker_url: amb4.service.consul:6667
+ connection: docker
+ roles:
+ - { role: bro, tags: ['bro'] }
+ - { role: snort, tags: ['snort'] }
+ - { role: yaf, tags: ['yaf'] }
+ - { role: pcap_replay, tags: ['pcap-replay'] }
+ - { role: sensor-test-mode, tags: ['sensor-test-mode'] }
+ tags:
+ - sensors
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/playbooks/metron_build.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/metron_build.yml b/metron-deployment/ansible/playbooks/metron_build.yml
new file mode 100644
index 0000000..83eb352
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/metron_build.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: all
+ become: false
+ roles:
+ - role: metron-builder
+ tags:
+ - build
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml
new file mode 100644
index 0000000..b517671
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/metron_full_install.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: all
+ pre_tasks:
+ - name: Verify Ansible Version
+ fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version is {{ ansible_version }}"
+ when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=')"
+
+- include: metron_build.yml
+ tags:
+ - build
+
+- include: ambari_install.yml
+ tags:
+ - ambari
+
+- include: metron_install.yml
+ tags:
+ - metron
+
+- include: sensor_install.yml
+
+#
+# deployment report
+#
+- hosts: metron
+ become: false
+ roles:
+ - { role: deployment-report }
+ tags:
+ - report
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/metron_install.yml b/metron-deployment/ansible/playbooks/metron_install.yml
new file mode 100644
index 0000000..49db5e6
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/metron_install.yml
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: ec2
+ become: true
+ tasks:
+ - include_vars: ../../amazon-ec2/conf/defaults.yml
+ tags:
+ - ec2
+
+- hosts: packer
+ become: true
+ tasks:
+ - include_vars: ../../development/centos6/ansible/inventory/group_vars/all
+ tags:
+ - packer
+
+#
+# all ambari slaves need the local repo setup, but most don't need the metron packages
+#
+- hosts: ambari_slave
+ become: true
+ roles:
+ - { role: metron-packages, packages_required: false }
+ tags:
+ - packages
+
+#
+# push the metron packages (RPM or DEB) to the metron host(s) only
+#
+- hosts: metron
+ become: true
+ roles:
+ - { role: metron-packages, packages_required: true }
+ tags:
+ - packages
+
+#
+# start installation of components in Ambari
+#
+- hosts: ambari_master
+ become: true
+ roles:
+ - role: ambari_config
+ tags:
+ - hdp-install
+ - hdp-deploy
+
+- hosts: ambari_master
+ become: true
+ roles:
+ - role: load_web_templates
+ tags:
+ - load_templates
+
+- hosts: pcap_server
+ become: true
+ roles:
+ - role: metron_pcapservice
+ tags:
+ - pcap-service
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/playbooks/sensor_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/playbooks/sensor_install.yml b/metron-deployment/ansible/playbooks/sensor_install.yml
new file mode 100644
index 0000000..a683ccb
--- /dev/null
+++ b/metron-deployment/ansible/playbooks/sensor_install.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: ec2
+ become: true
+ tasks:
+ - include_vars: ../../amazon-ec2/conf/defaults.yml
+ tags:
+ - ec2
+
+- hosts: packer
+ become: true
+ tasks:
+ - include_vars: ../../development/centos6/ansible/inventory/group_vars/all
+ tags:
+ - packer
+
+#
+# sensors
+#
+- hosts: sensors
+ become: true
+ roles:
+ - { role: ambari_gather_facts, tags: ['always'] }
+ - { role: tap_interface, tags: ['tap'] }
+ - { role: pycapa, tags: ['pycapa'] }
+ - { role: bro, tags: ['bro'] }
+ - { role: snort, tags: ['snort'] }
+ - { role: yaf, tags: ['yaf'] }
+ - { role: pcap_replay, tags: ['pcap-replay'] }
+ - { role: sensor-test-mode, tags: ['sensor-test-mode'] }
+ tags:
+ - sensors
+
+#
+# sensor stubs - rather than deploying the sensors, deploy ligher weight
+# stubs that mimic the behavior of the real sensors
+#
+- hosts: sensors
+ become: true
+ roles:
+ - { role: ambari_gather_facts }
+ - { role: sensor-stubs }
+ tags:
+ - sensor-stubs
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/README.md b/metron-deployment/ansible/roles/README.md
new file mode 100644
index 0000000..1cb5878
--- /dev/null
+++ b/metron-deployment/ansible/roles/README.md
@@ -0,0 +1,24 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Ansible Roles
+
+- Monit
+- OpenTaxii
+- Pcap Replay
+- Sensor Stubs
+- Sensor Test Mode
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
new file mode 100644
index 0000000..c04400e
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hadoop_logrotate_frequency: daily
+hadoop_logrotate_retention: 30
+centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo
+ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.4.2.0
+ubuntu_elasticsearch_packages_repo: https://artifacts.elastic.co/packages/5.x/apt
+ubuntu_elasticsearch_curator_repo: https://packages.elastic.co/curator/5/debian
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/meta/main.yml b/metron-deployment/ansible/roles/ambari_common/meta/main.yml
new file mode 100644
index 0000000..d7e46d6
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/meta/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - libselinux-python
+ - epel
+ - ntp
[02/50] [abbrv] metron git commit: METRON-1417: Disable pcap-service
by default in Monit (mmiklavc via mmiklavc) closes apache/metron#905
Posted by rm...@apache.org.
METRON-1417: Disable pcap-service by default in Monit (mmiklavc via mmiklavc) closes apache/metron#905
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/40411d4b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/40411d4b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/40411d4b
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 40411d4bdbb729d0e38e69a15a06970738a4b969
Parents: 9d582ea
Author: mmiklavc <mi...@gmail.com>
Authored: Wed Jan 24 15:33:08 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Wed Jan 24 15:33:08 2018 -0700
----------------------------------------------------------------------
metron-deployment/amazon-ec2/conf/defaults.yml | 1 -
.../inventory/full-dev-platform/group_vars/all | 1 -
.../roles/monit/tasks/monit-definitions.yml | 5 -----
.../monit/templates/monit/pcap-service.monit | 22 --------------------
4 files changed, 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/40411d4b/metron-deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/amazon-ec2/conf/defaults.yml b/metron-deployment/amazon-ec2/conf/defaults.yml
index ab5bfa9..124d7ab 100644
--- a/metron-deployment/amazon-ec2/conf/defaults.yml
+++ b/metron-deployment/amazon-ec2/conf/defaults.yml
@@ -17,7 +17,6 @@
---
# which services should be started?
services_to_start:
- - pcap-service
- yaf
- snort
- snort-logs
http://git-wip-us.apache.org/repos/asf/metron/blob/40411d4b/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
index 35fa154..c07278b 100644
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ b/metron-deployment/inventory/full-dev-platform/group_vars/all
@@ -17,7 +17,6 @@
# which services should be started?
services_to_start:
- - pcap-service
- snort
- snort-logs
- bro
http://git-wip-us.apache.org/repos/asf/metron/blob/40411d4b/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
index c9f5f07..13e00fc 100644
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ b/metron-deployment/roles/monit/tasks/monit-definitions.yml
@@ -20,8 +20,3 @@
template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit
when: ("sensors" in group_names) and (install_pcap_replay | default(False))
tags: sensors
-
-- name: Create monit definition for pcap-service
- template: src=monit/pcap-service.monit dest={{ monit_config_home }}/pcap-service.monit
- when: ("pcap_server" in group_names)
-
http://git-wip-us.apache.org/repos/asf/metron/blob/40411d4b/metron-deployment/roles/monit/templates/monit/pcap-service.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/pcap-service.monit b/metron-deployment/roles/monit/templates/monit/pcap-service.monit
deleted file mode 100644
index 3f39c99..0000000
--- a/metron-deployment/roles/monit/templates/monit/pcap-service.monit
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process pcap-service matching "org.apache.metron.pcapservice.rest.PcapService"
- start program = "/etc/init.d/pcapservice start"
- stop program = "/etc/init.d/pcapservice stop"
- if does not exist then restart
- group web
- group metron
[08/50] [abbrv] metron git commit: METRON-1430: Isolate jackson from
being used as arguments or returns from JSONUtils closes apache/metron#910
Posted by rm...@apache.org.
METRON-1430: Isolate jackson from being used as arguments or returns from JSONUtils closes apache/metron#910
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/878003a3
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/878003a3
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/878003a3
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 878003a3caf55380bd9e5898a9abe848586a278b
Parents: 92c2e22
Author: cstella <ce...@gmail.com>
Authored: Fri Jan 26 15:43:35 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Fri Jan 26 15:43:35 2018 -0500
----------------------------------------------------------------------
.../metron/maas/functions/MaaSFunctions.java | 4 +-
.../common/configuration/Configurations.java | 4 +-
.../configuration/ConfigurationsUtils.java | 5 +-
.../GlobalConfigurationOperations.java | 4 +-
.../IndexingConfigurationOperations.java | 4 +-
.../configuration/IndexingConfigurations.java | 4 +-
.../apache/metron/common/utils/JSONUtils.java | 60 ++++++++++++++++----
.../apache/metron/common/utils/KafkaUtils.java | 5 +-
.../configuration/StellarEnrichmentTest.java | 4 +-
.../metron/common/utils/JSONUtilsTest.java | 14 ++---
.../ZKConfigurationsCacheIntegrationTest.java | 10 ++--
.../TransformFilterExtractorDecorator.java | 4 +-
.../nonbulk/geo/GeoEnrichmentLoader.java | 4 +-
.../bulk/ElasticsearchImportExport.java | 4 +-
.../ElasticsearchSearchIntegrationTest.java | 6 ++
.../ElasticsearchUpdateIntegrationTest.java | 7 +--
.../components/ElasticSearchComponent.java | 4 +-
.../enrichment/cli/LatencySummarizer.java | 3 +-
.../integration/EnrichmentIntegrationTest.java | 6 +-
.../apache/metron/indexing/dao/HBaseDao.java | 3 +-
.../apache/metron/indexing/dao/IndexDao.java | 13 ++++-
.../metron/indexing/dao/update/Document.java | 4 +-
.../apache/metron/indexing/dao/InMemoryDao.java | 3 +-
.../indexing/dao/SearchIntegrationTest.java | 6 +-
.../HDFSIndexingIntegrationTest.java | 4 +-
.../integration/IndexingIntegrationTest.java | 3 +-
.../apache/metron/parsers/bolt/ParserBolt.java | 4 +-
.../metron/parsers/json/JSONMapParser.java | 4 +-
.../parsers/topology/ParserTopologyCLI.java | 4 +-
.../parsers/topology/config/ConfigHandlers.java | 4 +-
.../parsers/integration/ParserDriver.java | 4 +-
.../integration/WriterBoltIntegrationTest.java | 3 +-
.../common/benchmark/StellarMicrobenchmark.java | 4 +-
.../common/configuration/ConfigurationType.java | 4 +-
.../shell/DefaultStellarShellExecutor.java | 5 +-
.../stellar/common/shell/cli/StellarShell.java | 3 +-
.../metron/stellar/common/utils/JSONUtils.java | 43 ++++++++++++--
.../stellar/dsl/functions/StringFunctions.java | 27 +++++----
.../stellar/common/utils/JSONUtilsTest.java | 4 +-
.../dsl/functions/StringFunctionsTest.java | 11 ++--
40 files changed, 169 insertions(+), 147 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
index eacb64d..a260f25 100644
--- a/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
+++ b/metron-analytics/metron-maas-common/src/main/java/org/apache/metron/maas/functions/MaaSFunctions.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.maas.functions;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import java.lang.invoke.MethodHandles;
@@ -174,8 +173,7 @@ public class MaaSFunctions {
URL u = new URL(url + "/" + modelFunction);
String results = RESTUtil.INSTANCE.getRESTJSONResults(u, modelArgs);
- ret = JSONUtils.INSTANCE.load(results, new TypeReference<Map<String, Object>>() {
- });
+ ret = JSONUtils.INSTANCE.load(results, JSONUtils.MAP_SUPPLIER);
resultCache.put(cacheKey, ret);
return ret;
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
index ecf8a1b..af421a9 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/Configurations.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.common.configuration;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -61,8 +60,7 @@ public class Configurations implements Serializable {
}
public void updateGlobalConfig(InputStream io) throws IOException {
- Map<String, Object> globalConfig = JSONUtils.INSTANCE.load(io, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> globalConfig = JSONUtils.INSTANCE.load(io, JSONUtils.MAP_SUPPLIER);
updateGlobalConfig(globalConfig);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
index 5183788..a89db63 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigurationsUtils.java
@@ -596,10 +596,7 @@ public class ConfigurationsUtils {
byte[] patchData, CuratorFramework client) throws Exception {
byte[] configData = readConfigBytesFromZookeeper(configurationType, configName, client);
- JsonNode source = JSONUtils.INSTANCE.readTree(configData);
- JsonNode patch = JSONUtils.INSTANCE.readTree(patchData);
- JsonNode patchedConfig = JSONUtils.INSTANCE.applyPatch(patch, source);
- byte[] prettyPatchedConfig = JSONUtils.INSTANCE.toJSONPretty(patchedConfig);
+ byte[] prettyPatchedConfig = JSONUtils.INSTANCE.applyPatch(patchData, configData);
// ensure the patch produces a valid result; otherwise exception thrown during deserialization
String prettyPatchedConfigStr = new String(prettyPatchedConfig);
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java
index 4842c91..581ec3e 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/GlobalConfigurationOperations.java
@@ -18,7 +18,6 @@
package org.apache.metron.common.configuration;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
@@ -38,8 +37,7 @@ public class GlobalConfigurationOperations implements ConfigurationOperations {
@Override
public Object deserialize(String s) throws IOException {
- return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
- });
+ return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER);
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java
index a75c84e..7884e4d 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurationOperations.java
@@ -18,7 +18,6 @@
package org.apache.metron.common.configuration;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.util.Map;
import org.apache.curator.framework.CuratorFramework;
@@ -33,8 +32,7 @@ public class IndexingConfigurationOperations implements ConfigurationOperations
@Override
public Object deserialize(String s) throws IOException {
- return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
- });
+ return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER);
}
@Override
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
index 003b6df..7ef9f00 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/IndexingConfigurations.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.common.configuration;
-import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.common.utils.JSONUtils;
@@ -77,8 +76,7 @@ public class IndexingConfigurations extends Configurations {
}
public void updateSensorIndexingConfig(String sensorType, InputStream io) throws IOException {
- Map<String, Object> sensorIndexingConfig = JSONUtils.INSTANCE.load(io, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> sensorIndexingConfig = JSONUtils.INSTANCE.load(io, JSONUtils.MAP_SUPPLIER);
updateSensorIndexingConfig(sensorType, sensorIndexingConfig);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
index 280b167..135546d 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
@@ -29,6 +29,15 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+import com.google.common.reflect.TypeToken;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
@@ -36,6 +45,31 @@ import org.json.simple.parser.ParseException;
public enum JSONUtils {
INSTANCE;
+ public static class ReferenceSupplier<T> implements Supplier<TypeReference<T>> {
+ Type type;
+ protected ReferenceSupplier() {
+ Type superClass = this.getClass().getGenericSuperclass();
+ if(superClass instanceof Class) {
+ throw new IllegalArgumentException("Internal error: ReferenceSupplier constructed without actual type information");
+ } else {
+ this.type = ((ParameterizedType)superClass).getActualTypeArguments()[0];
+ }
+ }
+
+ @Override
+ public TypeReference<T> get() {
+ return new TypeReference<T>() {
+ @Override
+ public Type getType() {
+ return type;
+ }
+ };
+ }
+ }
+
+ public final static ReferenceSupplier<Map<String, Object>> MAP_SUPPLIER = new ReferenceSupplier<Map<String, Object>>() {};
+ public final static ReferenceSupplier<List<Object>> LIST_SUPPLIER = new ReferenceSupplier<List<Object>>(){};
+
private static ThreadLocal<JSONParser> _parser = ThreadLocal.withInitial(() ->
new JSONParser());
@@ -51,17 +85,17 @@ public enum JSONUtils {
}
- public <T> T load(InputStream is, TypeReference<T> ref) throws IOException {
- return _mapper.get().readValue(is, ref);
+ public <T> T load(InputStream is, ReferenceSupplier<T> ref) throws IOException {
+ return _mapper.get().readValue(is, (TypeReference<T>)ref.get());
}
- public <T> T load(String is, TypeReference<T> ref) throws IOException {
- return _mapper.get().readValue(is, ref);
+ public <T> T load(String is, ReferenceSupplier<T> ref) throws IOException {
+ return _mapper.get().readValue(is, (TypeReference<T>)ref.get());
}
- public <T> T load(File f, TypeReference<T> ref) throws IOException {
+ public <T> T load(File f, ReferenceSupplier<T> ref) throws IOException {
try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
- return _mapper.get().readValue(is, ref);
+ return _mapper.get().readValue(is, (TypeReference<T>)ref.get());
}
}
@@ -108,7 +142,7 @@ public enum JSONUtils {
* @param json JSON value to deserialize
* @return deserialized JsonNode Object
*/
- public JsonNode readTree(String json) throws IOException {
+ JsonNode readTree(String json) throws IOException {
return _mapper.get().readTree(json);
}
@@ -118,7 +152,7 @@ public enum JSONUtils {
* @param json JSON value to deserialize
* @return deserialized JsonNode Object
*/
- public JsonNode readTree(byte[] json) throws IOException {
+ JsonNode readTree(byte[] json) throws IOException {
return _mapper.get().readTree(json);
}
@@ -138,14 +172,16 @@ public enum JSONUtils {
* @param source Source JSON to apply patch to
* @return new json after applying the patch
*/
- public JsonNode applyPatch(String patch, String source) throws IOException {
+ public byte[] applyPatch(String patch, String source) throws IOException {
JsonNode patchNode = readTree(patch);
JsonNode sourceNode = readTree(source);
- return applyPatch(patchNode, sourceNode);
+ return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
}
- public JsonNode applyPatch(JsonNode patch, JsonNode source) throws IOException {
- return JsonPatch.apply(patch, source);
+ public byte[] applyPatch(byte[] patch, byte[] source) throws IOException {
+ JsonNode patchNode = readTree(patch);
+ JsonNode sourceNode = readTree(source);
+ return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
index a26a3bb..d54e2b8 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
@@ -18,7 +18,7 @@
package org.apache.metron.common.utils;
-import com.fasterxml.jackson.core.type.TypeReference;
+import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.curator.RetryPolicy;
@@ -48,8 +48,7 @@ public enum KafkaUtils {
for(String id : client.getChildren().forPath("/brokers/ids")) {
byte[] data = client.getData().forPath("/brokers/ids/" + id);
String brokerInfoStr = new String(data);
- Map<String, Object> brokerInfo = JSONUtils.INSTANCE.load(brokerInfoStr, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> brokerInfo = JSONUtils.INSTANCE.load(brokerInfoStr, JSONUtils.MAP_SUPPLIER);
String host = (String) brokerInfo.get("host");
if(host != null) {
ret.add(host + ":" + brokerInfo.get("port"));
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
index 8eb9bbc..60a7d82 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/StellarEnrichmentTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.common.configuration;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.ImmutableList;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.common.utils.JSONUtils;
@@ -221,8 +220,7 @@ public class StellarEnrichmentTest {
public static String message;
public static JSONObject getMessage() throws IOException {
- Map<String, Object> ret = JSONUtils.INSTANCE.load(message, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> ret = JSONUtils.INSTANCE.load(message, JSONUtils.MAP_SUPPLIER);
return new JSONObject(ret);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
index 7f4846e..4676591 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/utils/JSONUtilsTest.java
@@ -21,7 +21,6 @@ package org.apache.metron.common.utils;
import static org.hamcrest.CoreMatchers.equalTo;
import static org.junit.Assert.assertThat;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.File;
import java.io.IOException;
@@ -56,8 +55,7 @@ public class JSONUtilsTest {
put("b", "world");
}};
Map<String, Object> actual = JSONUtils.INSTANCE
- .load(configFile, new TypeReference<Map<String, Object>>() {
- });
+ .load(configFile, JSONUtils.MAP_SUPPLIER);
assertThat("config not equal", actual, equalTo(expected));
}
@@ -123,9 +121,8 @@ public class JSONUtilsTest {
@Test
public void applyPatch_modifies_source_json_doc() throws IOException {
- JsonNode actual = JSONUtils.INSTANCE.applyPatch(patchJson, sourceJson);
- JsonNode expected = JSONUtils.INSTANCE.readTree(expectedJson);
- assertThat(actual, equalTo(expected));
+ String actual = new String(JSONUtils.INSTANCE.applyPatch(patchJson, sourceJson));
+ assertThat(JSONUtils.INSTANCE.load(actual, JSONUtils.MAP_SUPPLIER), equalTo(JSONUtils.INSTANCE.load(expectedJson, JSONUtils.MAP_SUPPLIER)));
}
/**
@@ -160,9 +157,8 @@ public class JSONUtilsTest {
@Test
public void applyPatch_modifies_complex_source_json_doc() throws IOException {
- JsonNode actual = JSONUtils.INSTANCE.applyPatch(patchComplexJson, complexJson);
- JsonNode expected = JSONUtils.INSTANCE.readTree(expectedComplexJson);
- assertThat(actual, equalTo(expected));
+ String actual = new String(JSONUtils.INSTANCE.applyPatch(patchComplexJson, complexJson));
+ assertThat(JSONUtils.INSTANCE.load(actual, JSONUtils.MAP_SUPPLIER), equalTo(JSONUtils.INSTANCE.load(expectedComplexJson, JSONUtils.MAP_SUPPLIER)));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
index 64bf986..ec4a98a 100644
--- a/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
+++ b/metron-platform/metron-common/src/test/java/org/apache/metron/common/zookeeper/ZKConfigurationsCacheIntegrationTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.common.zookeeper;
-import com.fasterxml.jackson.core.type.TypeReference;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.commons.io.IOUtils;
import org.apache.curator.framework.CuratorFramework;
@@ -226,14 +225,14 @@ public class ZKConfigurationsCacheIntegrationTest {
ConfigurationsUtils.writeProfilerConfigToZookeeper( profilerConfig.getBytes(), client);
//indexing
{
- Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(testIndexingConfig, new TypeReference<Map<String, Object>>() {});
+ Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(testIndexingConfig, JSONUtils.MAP_SUPPLIER);
IndexingConfigurations config = cache.get( IndexingConfigurations.class);
assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorIndexingConfig("test")));
}
//enrichment
{
SensorEnrichmentConfig expectedConfig = JSONUtils.INSTANCE.load(testEnrichmentConfig, SensorEnrichmentConfig.class);
- Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfig, new TypeReference<Map<String, Object>>() {});
+ Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
EnrichmentConfigurations config = cache.get( EnrichmentConfigurations.class);
assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorEnrichmentConfig("test")));
assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
@@ -255,12 +254,11 @@ public class ZKConfigurationsCacheIntegrationTest {
@Test
public void validateBaseWrite() throws Exception {
File globalConfigFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/global.json");
- Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfigFile, new TypeReference<Map<String, Object>>() { });
+ Map<String, Object> expectedGlobalConfig = JSONUtils.INSTANCE.load(globalConfigFile, JSONUtils.MAP_SUPPLIER);
//indexing
{
File inFile = new File(TestConstants.SAMPLE_CONFIG_PATH + "/indexing/test.json");
- Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(inFile, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> expectedConfig = JSONUtils.INSTANCE.load(inFile, JSONUtils.MAP_SUPPLIER);
IndexingConfigurations config = cache.get( IndexingConfigurations.class);
assertEventually(() -> Assert.assertEquals(expectedConfig, config.getSensorIndexingConfig("test")));
assertEventually(() -> Assert.assertEquals(expectedGlobalConfig, config.getGlobalConfig()));
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
index c47dfc6..b665a01 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.dataloads.extractor;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
@@ -208,8 +207,7 @@ public class TransformFilterExtractorDecorator extends ExtractorDecorator implem
try {
return JSONUtils.INSTANCE.load(
new ByteArrayInputStream(ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(zkClient.get())),
- new TypeReference<Map<String, Object>>() {
- });
+ JSONUtils.MAP_SUPPLIER);
} catch (Exception e) {
LOG.warn("Exception thrown while attempting to get global config from Zookeeper.", e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java
index 322edec..b366015 100644
--- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java
+++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/geo/GeoEnrichmentLoader.java
@@ -18,7 +18,6 @@
package org.apache.metron.dataloads.nonbulk.geo;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import org.apache.commons.cli.*;
@@ -200,8 +199,7 @@ public class GeoEnrichmentLoader {
// Fetch the global configuration
Map<String, Object> global = JSONUtils.INSTANCE.load(
new ByteArrayInputStream(ConfigurationsUtils.readGlobalConfigBytesFromZookeeper(client)),
- new TypeReference<Map<String, Object>>() {
- });
+ JSONUtils.MAP_SUPPLIER);
// Update the global config and push it back
global.put(GeoLiteDatabase.GEO_HDFS_FILE, dstPath.toString() + "/" + srcPath.getName());
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
index 0a04dfc..77e0d61 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/bulk/ElasticsearchImportExport.java
@@ -18,7 +18,6 @@
package org.apache.metron.elasticsearch.bulk;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
@@ -71,8 +70,7 @@ public class ElasticsearchImportExport {
String line;
while ((line = br.readLine()) != null) {
Map<String, Object> inDoc = JSONUtils.INSTANCE
- .load(line, new TypeReference<Map<String, Object>>() {
- });
+ .load(line, JSONUtils.MAP_SUPPLIER);
Object id = inDoc.get("_id");
Object type = inDoc.get("_type");
String createRaw = String
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index f86a04d..1bc5b6e 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -21,13 +21,16 @@ package org.apache.metron.elasticsearch.integration;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.concurrent.ExecutionException;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
+import org.apache.metron.indexing.dao.search.GetRequest;
import org.apache.metron.integration.InMemoryComponent;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
@@ -37,6 +40,7 @@ import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
+import org.junit.Test;
public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
@@ -233,4 +237,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
throw new RuntimeException("Failed to index test data");
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
index 116ee4b..325d42e 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchUpdateIntegrationTest.java
@@ -18,7 +18,6 @@
package org.apache.metron.elasticsearch.integration;
import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Iterables;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
@@ -158,7 +157,7 @@ public class ElasticsearchUpdateIntegrationTest {
Assert.assertEquals(1, columns.size());
Assert.assertEquals(message0
, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue())
- , new TypeReference<Map<String, Object>>() {})
+ , JSONUtils.MAP_SUPPLIER)
);
}
{
@@ -195,10 +194,10 @@ public class ElasticsearchUpdateIntegrationTest {
NavigableMap<byte[], byte[]> columns = r.getFamilyMap(CF.getBytes());
Assert.assertEquals(2, columns.size());
Assert.assertEquals(message0, JSONUtils.INSTANCE.load(new String(columns.lastEntry().getValue())
- , new TypeReference<Map<String, Object>>() {})
+ , JSONUtils.MAP_SUPPLIER)
);
Assert.assertNotEquals(message0, JSONUtils.INSTANCE.load(new String(columns.firstEntry().getValue())
- , new TypeReference<Map<String, Object>>() {})
+ , JSONUtils.MAP_SUPPLIER)
);
}
{
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
index f95ef21..e716ce1 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/components/ElasticSearchComponent.java
@@ -19,7 +19,6 @@ package org.apache.metron.elasticsearch.integration.components;
import static java.util.Arrays.asList;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -210,8 +209,7 @@ public class ElasticSearchComponent implements InMemoryComponent {
indexRequestBuilder = indexRequestBuilder.setSource(doc);
Map<String, Object> esDoc = JSONUtils.INSTANCE
- .load(doc, new TypeReference<Map<String, Object>>() {
- });
+ .load(doc, JSONUtils.MAP_SUPPLIER);
indexRequestBuilder.setId((String) esDoc.get(Constants.GUID));
Object ts = esDoc.get("timestamp");
if (ts != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
index b40f2ad..2e5e8bf 100644
--- a/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
+++ b/metron-platform/metron-enrichment/src/main/java/org/apache/metron/enrichment/cli/LatencySummarizer.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.enrichment.cli;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
@@ -181,7 +180,7 @@ public class LatencySummarizer {
LatencyStats statsMap = new LatencyStats();
BufferedReader reader = new BufferedReader(new InputStreamReader(System.in));
for(String line = null;(line = reader.readLine()) != null;) {
- Map<String, Object> doc = JSONUtils.INSTANCE.load(line, new TypeReference<HashMap<String, Object>>() {});
+ Map<String, Object> doc = JSONUtils.INSTANCE.load(line, JSONUtils.MAP_SUPPLIER);
updateStats(statsMap, doc);
}
statsMap.printSummary(cmd.hasOption('m'));
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
index c457e86..828f4e3 100644
--- a/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
+++ b/metron-platform/metron-enrichment/src/test/java/org/apache/metron/enrichment/integration/EnrichmentIntegrationTest.java
@@ -24,7 +24,6 @@ import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAG
import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_RULE_SCORE;
import static org.apache.metron.enrichment.bolt.ThreatIntelJoinBolt.THREAT_TRIAGE_SCORE_KEY;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
@@ -163,8 +162,7 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
String globalConfigStr = null;
{
File globalConfig = new File(new File(TestConstants.SAMPLE_CONFIG_PATH), "global.json");
- Map<String, Object> config = JSONUtils.INSTANCE.load(globalConfig, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> config = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
config.put(SimpleHBaseEnrichmentFunctions.TABLE_PROVIDER_TYPE_CONF, MockHBaseTableProvider.class.getName());
config.put(SimpleHBaseEnrichmentFunctions.ACCESS_TRACKER_TYPE_CONF, "PERSISTENT_BLOOM");
config.put(PersistentBloomTrackerCreator.Config.PERSISTENT_BLOOM_TABLE, trackerHBaseTableName);
@@ -531,7 +529,7 @@ public class EnrichmentIntegrationTest extends BaseIntegrationTest {
, message -> {
try {
return new HashMap<>(JSONUtils.INSTANCE.load(new String(message)
- , new TypeReference<Map<String, Object>>() {}
+ , JSONUtils.MAP_SUPPLIER
)
);
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
index 72f2980..ebb9907 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/HBaseDao.java
@@ -18,7 +18,6 @@
package org.apache.metron.indexing.dao;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
@@ -210,7 +209,7 @@ public class HBaseDao implements IndexDao {
Long ts = Bytes.toLong(entry.getKey());
if(entry.getValue()!= null) {
Map<String, Object> json = JSONUtils.INSTANCE.load(new String(entry.getValue()),
- new TypeReference<Map<String, Object>>() {});
+ JSONUtils.MAP_SUPPLIER);
try {
Key k = Key.fromBytes(result.getRow());
return new Document(json, k.getGuid(), k.getSensorType(), ts);
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 03d348a..2c057d4 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -17,12 +17,16 @@
*/
package org.apache.metron.indexing.dao;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.flipkart.zjsonpatch.JsonPatch;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
@@ -42,6 +46,9 @@ import org.apache.metron.indexing.dao.update.ReplaceRequest;
*/
public interface IndexDao {
+ public static ThreadLocal<ObjectMapper> _mapper = ThreadLocal.withInitial(() ->
+ new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL));
+
/**
* Return search response based on the search request
*
@@ -140,9 +147,9 @@ public interface IndexDao {
throw new OriginalNotFoundException("Unable to patch an document that doesn't exist and isn't specified.");
}
}
- JsonNode originalNode = JSONUtils.INSTANCE.convert(latest, JsonNode.class);
- JsonNode patched = JSONUtils.INSTANCE.applyPatch(request.getPatch(), originalNode);
- Map<String, Object> updated = JSONUtils.INSTANCE.getMapper()
+ JsonNode originalNode = _mapper.get().convertValue(latest, JsonNode.class);
+ JsonNode patched = JsonPatch.apply(request.getPatch(), originalNode);
+ Map<String, Object> updated = _mapper.get()
.convertValue(patched, new TypeReference<Map<String, Object>>() {});
return new Document( updated
, request.getGuid()
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
index b5fcb02..6f2f779 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/Document.java
@@ -18,7 +18,6 @@
package org.apache.metron.indexing.dao.update;
-import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.metron.common.utils.JSONUtils;
import java.io.IOException;
@@ -47,8 +46,7 @@ public class Document {
}
private static Map<String, Object> convertDoc(String document) throws IOException {
- return JSONUtils.INSTANCE.load(document, new TypeReference<Map<String, Object>>() {
- });
+ return JSONUtils.INSTANCE.load(document, JSONUtils.MAP_SUPPLIER);
}
public String getSensorType() {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
index 002ec28..79c3bb3 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/InMemoryDao.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.indexing.dao;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Splitter;
import com.google.common.collect.ComparisonChain;
import com.google.common.collect.Iterables;
@@ -193,7 +192,7 @@ public class InMemoryDao implements IndexDao {
public static Map<String, Object> parse(String doc) {
try {
- return JSONUtils.INSTANCE.load(doc, new TypeReference<Map<String, Object>>() {});
+ return JSONUtils.INSTANCE.load(doc, JSONUtils.MAP_SUPPLIER);
} catch (IOException e) {
throw new IllegalStateException(e.getMessage(), e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index f381688..b40db46 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -18,7 +18,6 @@
*/
package org.apache.metron.indexing.dao;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -464,8 +463,7 @@ public abstract class SearchIntegrationTest {
@Test
public void get_all_latest_guid() throws Exception {
- List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>() {
- });
+ List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){});
Map<String, Document> docs = new HashMap<>();
for(Document doc : dao.getAllLatest(request)) {
@@ -945,4 +943,4 @@ public abstract class SearchIntegrationTest {
protected abstract IndexDao createDao() throws Exception;
protected abstract InMemoryComponent startIndex() throws Exception;
protected abstract void loadTestData() throws Exception;
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
index ab5cc3f..b8af6a3 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/HDFSIndexingIntegrationTest.java
@@ -18,7 +18,6 @@
package org.apache.metron.indexing.integration;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import org.apache.metron.common.interfaces.FieldNameConverter;
@@ -75,8 +74,7 @@ public class HDFSIndexingIntegrationTest extends IndexingIntegrationTest {
Iterables.addAll(ret, Iterables.transform(data, bytes -> {
String s = new String(bytes);
try {
- return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
- });
+ return JSONUtils.INSTANCE.load(s,JSONUtils.MAP_SUPPLIER);
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
index b0b6cc2..1671ab3 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/integration/IndexingIntegrationTest.java
@@ -18,7 +18,6 @@
package org.apache.metron.indexing.integration;
-import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.curator.framework.CuratorFramework;
import org.apache.metron.TestConstants;
import org.apache.metron.common.Constants;
@@ -81,7 +80,7 @@ public abstract class IndexingIntegrationTest extends BaseIntegrationTest {
}});
List<Map<String, Object>> inputDocs = new ArrayList<>();
for(byte[] b : inputMessages) {
- Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b), new TypeReference<Map<String, Object>>() {});
+ Map<String, Object> m = JSONUtils.INSTANCE.load(new String(b), JSONUtils.MAP_SUPPLIER);
inputDocs.add(m);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
index 99785b2..6fc4ed7 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/bolt/ParserBolt.java
@@ -19,7 +19,6 @@ package org.apache.metron.parsers.bolt;
import static org.apache.metron.common.Constants.METADATA_PREFIX;
-import com.fasterxml.jackson.core.type.TypeReference;
import java.io.IOException;
import java.io.Serializable;
import java.lang.invoke.MethodHandles;
@@ -146,8 +145,7 @@ public class ParserBolt extends ConfiguredParserBolt implements Serializable {
try {
keyStr = keyObj == null?null:new String(keyObj);
if(!StringUtils.isEmpty(keyStr)) {
- Map<String, Object> metadata = JSONUtils.INSTANCE.load(keyStr, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> metadata = JSONUtils.INSTANCE.load(keyStr,JSONUtils.MAP_SUPPLIER);
for(Map.Entry<String, Object> kv : metadata.entrySet()) {
ret.put(METADATA_PREFIX + kv.getKey(), kv.getValue());
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
index 5d82456..7e5468f 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/json/JSONMapParser.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.parsers.json;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import org.apache.metron.common.utils.JSONUtils;
@@ -96,8 +95,7 @@ public class JSONMapParser extends BasicParser {
try {
String originalString = new String(rawMessage);
//convert the JSON blob into a String -> Object map
- Map<String, Object> rawMap = JSONUtils.INSTANCE.load(originalString, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> rawMap = JSONUtils.INSTANCE.load(originalString, JSONUtils.MAP_SUPPLIER);
JSONObject ret = normalizeJSON(rawMap);
ret.put("original_string", originalString );
if(!ret.containsKey("timestamp")) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
index 4ce0508..3824212 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyCLI.java
@@ -24,7 +24,6 @@ import org.apache.storm.Config;
import org.apache.storm.LocalCluster;
import org.apache.storm.StormSubmitter;
import org.apache.storm.utils.Utils;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import org.apache.commons.cli.*;
import org.apache.commons.io.FileUtils;
@@ -492,8 +491,7 @@ public class ParserTopologyCLI {
throw new IllegalArgumentException("Unable to load JSON file at " + inputFile.getAbsolutePath());
}
try {
- return JSONUtils.INSTANCE.load(json, new TypeReference<Map<String, Object>>() {
- });
+ return JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER);
} catch (IOException e) {
throw new IllegalStateException("Unable to process JSON.", e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
index f6c04a6..8d98922 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/config/ConfigHandlers.java
@@ -19,7 +19,6 @@
package org.apache.metron.parsers.topology.config;
import org.apache.storm.Config;
-import com.fasterxml.jackson.core.type.TypeReference;
import org.apache.commons.io.FileUtils;
import org.apache.metron.common.utils.JSONUtils;
@@ -81,8 +80,7 @@ public class ConfigHandlers {
json = arg.getArg();
}
try {
- arg.getConfig().putAll(JSONUtils.INSTANCE.load(json, new TypeReference<Map<String, Object>>() {
- }));
+ arg.getConfig().putAll(JSONUtils.INSTANCE.load(json, JSONUtils.MAP_SUPPLIER));
} catch (IOException e) {
throw new IllegalStateException("Unable to process JSON snippet.", e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
index b844104..deb0217 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/integration/ParserDriver.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.parsers.integration;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.ImmutableList;
import org.apache.metron.common.configuration.ConfigurationsUtils;
import org.apache.metron.common.configuration.FieldValidator;
@@ -147,8 +146,7 @@ public class ParserDriver {
public ParserDriver(String sensorType, String parserConfig, String globalConfig) throws IOException {
config = SensorParserConfig.fromBytes(parserConfig.getBytes());
this.sensorType = sensorType;
- this.globalConfig = JSONUtils.INSTANCE.load(globalConfig, new TypeReference<Map<String, Object>>() {
- });
+ this.globalConfig = JSONUtils.INSTANCE.load(globalConfig, JSONUtils.MAP_SUPPLIER);
}
public ProcessorResult<List<byte[]>> run(List<byte[]> in) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
index d4d1c02..0cfaae3 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/writers/integration/WriterBoltIntegrationTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.writers.integration;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import org.adrianwalker.multilinestring.Multiline;
@@ -171,7 +170,7 @@ public class WriterBoltIntegrationTest extends BaseIntegrationTest {
, message -> {
try {
return new JSONObject(JSONUtils.INSTANCE.load(new String(message)
- , new TypeReference<Map<String, Object>>() {}
+ ,JSONUtils.MAP_SUPPLIER
)
);
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java
index 35fa673..914105e 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/benchmark/StellarMicrobenchmark.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.stellar.common.benchmark;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.io.Files;
@@ -231,8 +230,7 @@ public class StellarMicrobenchmark {
List<String> lines = Files.readLines(expressionsFile, Charset.defaultCharset());
Map<String, Object> variables = new HashMap<>();
if(variablesFile.isPresent()) {
- variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), new TypeReference<Map<String, Object>>() {
- });
+ variables = JSONUtils.INSTANCE.load(new FileInputStream(variablesFile.get()), JSONUtils.MAP_SUPPLIER);
}
int numTimes = DEFAULT_NUM_TIMES;
if(BenchmarkOptions.NUM_TIMES.has(cli)) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java
index 16cde83..99859b2 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/configuration/ConfigurationType.java
@@ -18,7 +18,6 @@
package org.apache.metron.stellar.common.configuration;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Function;
import org.apache.metron.stellar.common.Constants;
import org.apache.metron.stellar.common.utils.JSONUtils;
@@ -30,8 +29,7 @@ public enum ConfigurationType implements Function<String, Object> {
GLOBAL("global",".", s -> {
try {
- return JSONUtils.INSTANCE.load(s, new TypeReference<Map<String, Object>>() {
- });
+ return JSONUtils.INSTANCE.load(s, JSONUtils.MAP_SUPPLIER);
} catch (IOException e) {
throw new RuntimeException("Unable to load " + s, e);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
index f83bb9e..3f2c495 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/DefaultStellarShellExecutor.java
@@ -19,7 +19,6 @@
*/
package org.apache.metron.stellar.common.shell;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.collect.Maps;
import org.apache.commons.collections.map.UnmodifiableMap;
import org.apache.commons.lang3.StringUtils;
@@ -370,9 +369,7 @@ public class DefaultStellarShellExecutor implements StellarShellExecutor {
*/
private Map<String, Object> fetchGlobalConfig(CuratorFramework zkClient) throws Exception {
byte[] raw = readGlobalConfigBytesFromZookeeper(zkClient);
- return JSONUtils.INSTANCE.load(
- new ByteArrayInputStream(raw),
- new TypeReference<Map<String, Object>>() {});
+ return JSONUtils.INSTANCE.load( new ByteArrayInputStream(raw), JSONUtils.MAP_SUPPLIER);
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java
index 44ad28c..c81df61 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/StellarShell.java
@@ -20,7 +20,6 @@
package org.apache.metron.stellar.common.shell.cli;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
import org.apache.commons.cli.CommandLine;
@@ -221,7 +220,7 @@ public class StellarShell extends AeshConsoleCallback implements Completion {
String variablePath = commandLine.getOptionValue("v");
Map<String, Object> variables = JSONUtils.INSTANCE.load(
new File(variablePath),
- new TypeReference<Map<String, Object>>() {});
+ JSONUtils.MAP_SUPPLIER);
// for each variable...
for(Map.Entry<String, Object> kv : variables.entrySet()) {
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java
index d7e90cf..9fb1c3f 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/utils/JSONUtils.java
@@ -27,6 +27,12 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
@@ -35,23 +41,48 @@ import org.json.simple.parser.ParseException;
public enum JSONUtils {
INSTANCE;
+ public static class ReferenceSupplier<T> implements Supplier<TypeReference<T>> {
+ Type type;
+ protected ReferenceSupplier() {
+ Type superClass = this.getClass().getGenericSuperclass();
+ if(superClass instanceof Class) {
+ throw new IllegalArgumentException("Internal error: ReferenceSupplier constructed without actual type information");
+ } else {
+ this.type = ((ParameterizedType)superClass).getActualTypeArguments()[0];
+ }
+ }
+
+ @Override
+ public TypeReference<T> get() {
+ return new TypeReference<T>() {
+ @Override
+ public Type getType() {
+ return type;
+ }
+ };
+ }
+ }
+
+ public final static ReferenceSupplier<Map<String, Object>> MAP_SUPPLIER = new ReferenceSupplier<Map<String, Object>>(){};
+ public final static ReferenceSupplier<List<Object>> LIST_SUPPLIER = new ReferenceSupplier<List<Object>>(){};
+
private static ThreadLocal<JSONParser> _parser = ThreadLocal.withInitial(() ->
new JSONParser());
private static ThreadLocal<ObjectMapper> _mapper = ThreadLocal.withInitial(() ->
new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL));
- public <T> T load(InputStream is, TypeReference<T> ref) throws IOException {
- return _mapper.get().readValue(is, ref);
+ public <T> T load(InputStream is, ReferenceSupplier<T> ref) throws IOException {
+ return _mapper.get().readValue(is, (TypeReference<T>)ref.get());
}
- public <T> T load(String is, TypeReference<T> ref) throws IOException {
- return _mapper.get().readValue(is, ref);
+ public <T> T load(String is, ReferenceSupplier<T> ref) throws IOException {
+ return _mapper.get().readValue(is, (TypeReference<T>)ref.get());
}
- public <T> T load(File f, TypeReference<T> ref) throws IOException {
+ public <T> T load(File f, ReferenceSupplier<T> ref) throws IOException {
try (InputStream is = new BufferedInputStream(new FileInputStream(f))) {
- return _mapper.get().readValue(is, ref);
+ return _mapper.get().readValue(is, (TypeReference<T>)ref.get());
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
index 4dc4790..2865ea7 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/StringFunctions.java
@@ -20,7 +20,6 @@ package org.apache.metron.stellar.dsl.functions;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.core.type.TypeReference;
import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.Iterables;
@@ -569,15 +568,18 @@ public class StringFunctions {
throw new ParseException("Valid JSON string not supplied");
}
// Return parsed JSON Object as a HashMap
+ String in = (String)strings.get(0);
try {
- return JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<Map<String, Object>>(){});
+ return (Map)JSONUtils.INSTANCE.load(in, JSONUtils.MAP_SUPPLIER);
} catch (JsonProcessingException ex) {
- throw new ParseException("Valid JSON string not supplied", ex);
- } catch (IOException e) {
- e.printStackTrace();
+ throw new ParseException(String.format("%s is not a valid JSON string", in), ex);
+ } catch (IOException ex) {
+ throw new ParseException(String.format("%s is not a valid JSON string", in), ex);
+ }
+ catch (ClassCastException ex) {
+ throw new ParseException(String.format("%s is not a valid JSON string, expected a map", in), ex);
}
}
- return new ParseException("Unable to parse JSON string");
}
}
@@ -606,13 +608,16 @@ public class StringFunctions {
throw new ParseException("Valid JSON string not supplied");
}
// Return parsed JSON Object as a List
+ String in = (String)strings.get(0);
try {
- return (List) JSONUtils.INSTANCE.load((String) strings.get(0), new TypeReference<List<Object>>(){});
+ return (List) JSONUtils.INSTANCE.load(in, JSONUtils.LIST_SUPPLIER);
} catch (JsonProcessingException ex) {
- throw new ParseException("Valid JSON string not supplied", ex);
- } catch (IOException e) {
- e.printStackTrace();
- throw new ParseException("Valid JSON string not supplied", e);
+ throw new ParseException(String.format("%s is not a valid JSON string", in), ex);
+ } catch (IOException ex) {
+ throw new ParseException(String.format("%s is not a valid JSON string", in), ex);
+ }
+ catch (ClassCastException ex) {
+ throw new ParseException(String.format("%s is not a valid JSON string, expected a list", in), ex);
}
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/JSONUtilsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/JSONUtilsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/JSONUtilsTest.java
index f4692e3..e28bfb7 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/JSONUtilsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/common/utils/JSONUtilsTest.java
@@ -17,7 +17,6 @@
*/
package org.apache.metron.stellar.common.utils;
-import com.fasterxml.jackson.core.type.TypeReference;
import org.adrianwalker.multilinestring.Multiline;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -54,8 +53,7 @@ public class JSONUtilsTest {
put("a", "hello");
put("b", "world");
}};
- Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, new TypeReference<Map<String, Object>>() {
- });
+ Map<String, Object> actual = JSONUtils.INSTANCE.load(configFile, JSONUtils.MAP_SUPPLIER);
Assert.assertThat("config not equal", actual, equalTo(expected));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/878003a3/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
index 418bf2d..9f78fe1 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/StringFunctionsTest.java
@@ -617,7 +617,8 @@ public class StringFunctionsTest {
//Simple Arrays
boolean thrown = false;
try {
- run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3));
+ Object o = run("TO_JSON_MAP(msg)", ImmutableMap.of("msg", string3));
+ System.out.println(string3 + " == " + o);
} catch (ParseException pe) {
thrown = true;
}
@@ -656,7 +657,7 @@ public class StringFunctionsTest {
run("TO_JSON_MAP('123, 456')", new HashedMap<>());
} catch (ParseException pe) {
thrown = true;
- Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ Assert.assertTrue(pe.getMessage().contains("is not a valid JSON string"));
}
Assert.assertTrue(thrown);
thrown = false;
@@ -666,7 +667,7 @@ public class StringFunctionsTest {
run("TO_JSON_MAP('{\"foo\" : 2')", new HashedMap<>());
} catch (ParseException pe) {
thrown = true;
- Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ Assert.assertTrue(pe.getMessage().contains("is not a valid JSON string"));
}
Assert.assertTrue(thrown);
thrown = false;
@@ -730,7 +731,7 @@ public class StringFunctionsTest {
run("TO_JSON_LIST('123, 456')", new HashedMap<>());
} catch (ParseException pe) {
thrown = true;
- Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ Assert.assertTrue(pe.getMessage().contains("is not a valid JSON string"));
}
Assert.assertTrue(thrown);
@@ -740,7 +741,7 @@ public class StringFunctionsTest {
run("TO_JSON_LIST('{\"foo\" : 2')", new HashedMap<>());
} catch (ParseException pe) {
thrown = true;
- Assert.assertTrue(pe.getMessage().contains("Valid JSON string not supplied"));
+ Assert.assertTrue(pe.getMessage().contains("is not a valid JSON string"));
}
Assert.assertTrue(thrown);
}
[47/50] [abbrv] metron git commit: METRON-1386: Fix Metron Website
Required Links closes apache/incubator-metron#935
Posted by rm...@apache.org.
METRON-1386: Fix Metron Website Required Links closes apache/incubator-metron#935
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fa5cff2c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fa5cff2c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fa5cff2c
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: fa5cff2c3534c5d30f265b0118faffe971e5063b
Parents: e59059b
Author: Anand Subramanian <an...@gmail.com>
Authored: Thu Feb 15 15:32:37 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Feb 15 15:32:37 2018 -0500
----------------------------------------------------------------------
site/_includes/footer.html | 3 +-
site/_includes/primary-nav-items.html | 2 +
site/asf/index.md | 90 ++++++++++++++++++++++++++++++
site/community/index.md | 4 ++
4 files changed, 98 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/_includes/footer.html
----------------------------------------------------------------------
diff --git a/site/_includes/footer.html b/site/_includes/footer.html
index 55c47a9..07615f4 100644
--- a/site/_includes/footer.html
+++ b/site/_includes/footer.html
@@ -1,6 +1,7 @@
</section>
<section class="intro-block darken" id="docshome">
<div class="content-960 hover-btn text-center disclaimer">
- The contents of this website are © 2016 Apache Software Foundation under the terms of the Apache License v2. Apache Metron and its logo are trademarks of the Apache Software Foundation.
+ <p>Copyright © 2018, The Apache Software Foundation. Licensed under the <a class="white" href="http://www.apache.org/licenses/LICENSE-2.0">Apache License, Version 2.0</a>.</p>
+ <p>Apache Metron and its logo are trademarks of the The Apache Software Foundation.</p>
</div>
</section>
http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/_includes/primary-nav-items.html
----------------------------------------------------------------------
diff --git a/site/_includes/primary-nav-items.html b/site/_includes/primary-nav-items.html
index 2799976..3b97c63 100644
--- a/site/_includes/primary-nav-items.html
+++ b/site/_includes/primary-nav-items.html
@@ -9,6 +9,8 @@
</li>
<li class="training-menu"><a href="/news/">News</a>
</li>
+ <li class="training-menu"><a href="/asf/">ASF</a>
+ </li>
<li>
<button class="button-default button-green" style="line-height:43px;"> <a>Download </a></button>
</li>
http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/asf/index.md
----------------------------------------------------------------------
diff --git a/site/asf/index.md b/site/asf/index.md
new file mode 100644
index 0000000..ca98112
--- /dev/null
+++ b/site/asf/index.md
@@ -0,0 +1,90 @@
+---
+layout: default
+title: Apache Software Foundation
+---
+
+<section class="hero-second-level no-padding">
+ <div class="bg-img">
+ <img src="/img/metron_datacenter.jpg" alt="UNLOCK THE POWER OF YOUR DATA" style="width: 100%; left: 0px;">
+ </div>
+ <div class="v-middle-wrapper">
+ <div class="v-middle-inner">
+ <div class="v-middle">
+ <h1>real-time big data security </h1>
+ </div>
+ </div>
+ </div>
+</section>
+
+<section class="no-padding">
+ <div class="fixed-anchor" style="min-height: 99px;">
+ <div class="fixed-links" style="top: 63px; transition: top 0.5s ease;">
+ <ul>
+ <li class="active"><a href="#home">Home</a></li>
+ <li class=""><a href="#license">License</a></li>
+ <li class=""><a href="#events">Events</a></li>
+ <li class=""><a href="#sponsorship">Sponsorship</a></li>
+ <li class=""><a href="#security">Security</a></li>
+ <li class=""><a href="#thanks">Thanks</a></li>
+ </ul>
+ </div>
+ </div>
+</section>
+
+<section class="events-section feature body-content" id="home">
+<div class="text-center">
+ <h2>Home</h2>
+</div>
+ <div class="content-960 hover-btn text-center">
+ To know more about The Apache Software Foundation, visit:
+ </div>
+ <div class="content-960 hover-btn text-center">
+ <a class="button-default" href="https://www.apache.org" target="_blank">APACHE</a>
+ </div>
+</section>
+
+<section class="intro-block darken" id="license">
+ <div class="text-center">
+ <h2>License</h2>
+ </div>
+ <div class="content-960 hover-btn text-center">
+ <a class="button-default" href="https://www.apache.org/licenses/" target="_blank">APACHE LICENSE</a>
+ </div>
+</section>
+
+<section class="intro-block darken" id="events">
+ <div class="text-center">
+ <h2>Events</h2>
+ </div>
+ <div class="content-960 hover-btn text-center">
+ <p>To know more about latest happenings, go to:</p>
+ <a class="button-default" href="https://www.apache.org/events/current-event" target="_blank">EVENTS</a>
+ </div>
+</section>
+
+<section class="intro-block darken" id="sponsorship">
+ <div class="text-center">
+ <h2>Sponsorship</h2>
+ </div>
+ <div class="content-960 hover-btn text-center">
+ <a class="button-default" href="https://www.apache.org/foundation/sponsorship.html" target="_blank">SPONSORSHIP</a>
+ </div>
+</section>
+
+<section class="intro-block darken" id="security">
+ <div class="text-center">
+ <h2>Security</h2>
+ </div>
+ <div class="content-960 hover-btn text-center">
+ <a class="button-default" href="https://www.apache.org/security/" target="_blank">SECURITY</a>
+ </div>
+</section>
+
+<section class="intro-block darken" id="thanks">
+ <div class="text-center">
+ <h2>Thanks</h2>
+ </div>
+ <div class="content-960 hover-btn text-center">
+ <a class="button-default" href="https://www.apache.org/foundation/thanks.html" target="_blank">THANKS</a>
+ </div>
+</section>
http://git-wip-us.apache.org/repos/asf/metron/blob/fa5cff2c/site/community/index.md
----------------------------------------------------------------------
diff --git a/site/community/index.md b/site/community/index.md
index e848748..7c09b14 100644
--- a/site/community/index.md
+++ b/site/community/index.md
@@ -85,6 +85,10 @@ title: Apache Metron Community
<td style="text-align: left">Jon Zeolla</td> <td style="text-align: left">jonzeolla</td> <td style="text-align: left">Committer</td> </tr>
<tr>
<td style="text-align: left">JJ Meyer</td> <td style="text-align: left">jmeyer</td> <td style="text-align: left">Committer</td> </tr>
+ <tr>
+ <td style="text-align: left">Raghu Mitra</td> <td style="text-align: left">iraghumitra</td> <td style="text-align: left">Committer</td> </tr>
+ <tr>
+ <td style="text-align: left">Anand Subramanian</td> <td style="text-align: left">anandsubbu</td> <td style="text-align: left">Committer</td> </tr>
</tbody>
</table>
</section>
[31/50] [abbrv] metron git commit: METRON-1389 Zeppelin notebook
import does not work with Ambari 2.6 (anandsubbu) closes apache/metron#888
Posted by rm...@apache.org.
METRON-1389 Zeppelin notebook import does not work with Ambari 2.6 (anandsubbu) closes apache/metron#888
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1c9437c4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1c9437c4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1c9437c4
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 1c9437c41c4eb626bd44193ec6545f9356f3f7b2
Parents: a285b83
Author: anandsubbu <an...@gmail.com>
Authored: Tue Jan 30 20:39:45 2018 +0530
Committer: anandsubbu <an...@apache.org>
Committed: Tue Jan 30 20:39:45 2018 +0530
----------------------------------------------------------------------
.../package/scripts/indexing_commands.py | 48 ++++++++++++++++++++
.../CURRENT/package/scripts/indexing_master.py | 14 ++++--
.../package/scripts/params/params_linux.py | 1 +
.../package/scripts/params/status_params.py | 1 +
4 files changed, 61 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index c057b72..4c862f0 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -16,12 +16,15 @@ limitations under the License.
"""
import os
+import re
+import requests
import time
from datetime import datetime
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute, File
+from resource_management.libraries.functions import format as ambari_format
import metron_service
import metron_security
@@ -327,3 +330,48 @@ class IndexingCommands:
raise Fail("Indexing topology not running")
Logger.info("Indexing service check completed successfully")
+
+ def get_zeppelin_auth_details(self, ses, zeppelin_server_url, env):
+ """
+ With Ambari 2.5+, Zeppelin server is enabled to work with Shiro authentication, which requires user/password
+ for authentication (see https://zeppelin.apache.org/docs/0.6.0/security/shiroauthentication.html for details).
+
+ This method checks if Shiro authentication is enabled on the Zeppelin server. And if enabled, it returns the
+ session connection details to be used for importing Zeppelin notebooks.
+ :param ses: Session handle
+ :param zeppelin_server_url: Zeppelin Server URL
+ :return: ses
+ """
+ from params import params
+ env.set_params(params)
+
+ # Check if authentication is enabled on the Zeppelin server
+ try:
+ ses.get(ambari_format('http://{zeppelin_server_url}/api/login'))
+
+ # Establish connection if authentication is enabled
+ try:
+ Logger.info("Shiro authentication is found to be enabled on the Zeppelin server.")
+ # Read the Shiro admin user credentials from Zeppelin config in Ambari
+ seen_users = False
+ username = None
+ password = None
+ if re.search(r'^\[users\]', params.zeppelin_shiro_ini_content, re.MULTILINE):
+ seen_users = True
+ tokens = re.search(r'^admin\ =.*', params.zeppelin_shiro_ini_content, re.MULTILINE).group()
+ userpassword = tokens.split(',')[0].strip()
+ username = userpassword.split('=')[0].strip()
+ password = userpassword.split('=')[1].strip()
+ else:
+ Logger.error("ERROR: Admin credentials config was not found in shiro.ini. Notebook import may fail.")
+
+ zeppelin_payload = {'userName': username, 'password' : password}
+ ses.post(ambari_format('http://{zeppelin_server_url}/api/login'), data=zeppelin_payload)
+ except:
+ pass
+
+ # If authentication is not enabled, fall back to default method of imporing notebooks
+ except requests.exceptions.RequestException:
+ ses.get(ambari_format('http://{zeppelin_server_url}/api/notebook'))
+
+ return ses
http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 18d5224..ce8c074 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -15,6 +15,7 @@ limitations under the License.
"""
import os
+import requests
from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute
@@ -151,14 +152,21 @@ class Indexing(Script):
def zeppelin_notebook_import(self, env):
from params import params
env.set_params(params)
+ commands = IndexingCommands(params)
Logger.info(ambari_format('Searching for Zeppelin Notebooks in {metron_config_zeppelin_path}'))
+
+ # Check if authentication is configured on Zeppelin server, and fetch details if enabled.
+ ses = requests.session()
+ ses = commands.get_zeppelin_auth_details(ses, params.zeppelin_server_url, env)
for dirName, subdirList, files in os.walk(params.metron_config_zeppelin_path):
for fileName in files:
if fileName.endswith(".json"):
- zeppelin_cmd = ambari_format(
- 'curl -s -XPOST http://{zeppelin_server_url}/api/notebook/import -d "@' + os.path.join(dirName, fileName) + '"')
- Execute(zeppelin_cmd, logoutput=True)
+ Logger.info("Importing notebook: " + fileName)
+ zeppelin_import_url = ambari_format('http://{zeppelin_server_url}/api/notebook/import')
+ zeppelin_notebook = {'file' : open(os.path.join(dirName, fileName), 'rb')}
+ res = ses.post(zeppelin_import_url, files=zeppelin_notebook)
+ Logger.info("Result: " + res.text)
if __name__ == "__main__":
Indexing().execute()
http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index 0d5b721..3506fab 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -202,6 +202,7 @@ meta_index_path = tmp_dir + "/metaalert_index.template"
# Zeppelin Notebooks
metron_config_zeppelin_path = format("{metron_config_path}/zeppelin")
+zeppelin_shiro_ini_content = status_params.zeppelin_shiro_ini_content
# kafka_security
kafka_security_protocol = config['configurations']['kafka-broker'].get('security.inter.broker.protocol', 'PLAINTEXT')
http://git-wip-us.apache.org/repos/asf/metron/blob/1c9437c4/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
index b43c30c..62cfc7a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py
@@ -95,6 +95,7 @@ storm_rest_addr = config['configurations']['metron-env']['storm_rest_addr']
# Zeppelin
zeppelin_server_url = config['configurations']['metron-env']['zeppelin_server_url']
+zeppelin_shiro_ini_content = config['configurations']['zeppelin-shiro-ini']['shiro_ini_content']
# Security
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
[20/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml
new file mode 100644
index 0000000..c333025
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create home directory
+ file: path={{ item }} state=directory mode=0755
+ with_items:
+ - "{{ sensor_stubs_home }}"
+ - "{{ sensor_stubs_bin }}"
+ - "{{ sensor_stubs_data }}"
+
+- name: Distribute stub data
+ copy: src={{ item }} dest={{ sensor_stubs_data }}/
+ with_items:
+ - bro.out
+ - snort.out
+ - yaf.out
+
+- name: Install service script
+ template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755
+
+- name: Install sensor stubs
+ template: src={{ item }} dest={{ sensor_stubs_bin }}/ mode=0755
+ with_items:
+ - start-bro-stub
+ - start-snort-stub
+ - start-yaf-stub
+
+- name: Start sensor stubs
+ service: name=sensor-stubs state=restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs
new file mode 100644
index 0000000..eaab69d
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs
@@ -0,0 +1,154 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron sensor-stubs service
+# chkconfig: 345 20 80
+# description: Simulates the behavior of a sensor by sending canned telemetry data to a Kafka topic
+# processname: sensor-stubs
+#
+NAME=sensor-stubs
+DESC="Simulates the behavior of a sensor by sending canned telemetry data to a Kafka topic"
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="{{ sensor_stubs_log }}"
+DAEMON_PATH="{{ sensor_stubs_home }}"
+DATA="{{ sensor_stubs_data }}"
+
+# the delay between each 'batch' of messages in seconds.
+# see {{ sensor_stubs_bin }}/start-sensor-stub for more information.
+DELAY={{ sensor_stubs_delay }}
+
+# the number of messages to send in each batch.
+# see {{ sensor_stubs_bin }}/start-sensor-stub for more information.
+COUNT={{ sensor_stubs_count }}
+
+#
+# which sensors? defaults to bro, snort, yaf
+#
+SENSORS="${@:2}"
+if [ -z "${SENSORS}" ]; then
+ SENSORS=('bro' 'yaf' 'snort')
+fi
+
+#
+# start a sensor stub
+#
+start() {
+
+ # if pidfile exists, do not start another
+ PIDFILE="/var/run/$NAME-$1.pid"
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ printf "%10s: %s \n" "$1" "OK [$PID]"
+ return
+ fi
+
+ # kick-off the daemon
+ DAEMON="{{ sensor_stubs_bin }}/start-$1-stub $DELAY $COUNT"
+ PID=`$DAEMON >> $LOGFILE 2>&1 & echo $!`
+
+ if [ -z $PID ]; then
+ printf "%10s: %s \n" "$1" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%10s: %s \n" "$1" "Ok [$PID]"
+ fi
+}
+
+#
+# stop a sensor stub
+#
+stop() {
+ PIDFILE="/var/run/$NAME-$1.pid"
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ while sleep 1
+ echo -n "."
+ kill -0 $PID >/dev/null 2>&1
+ do
+ kill $PID
+ done
+
+ printf "%10s: %s \n" "$1" "Stopped [$PID]"
+ rm -f $PIDFILE
+ else
+ printf "%10s: %s \n" "$1" "Not running"
+ fi
+}
+
+#
+# status check of sensor stub
+#
+status() {
+ PIDFILE="/var/run/$NAME-$1.pid"
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%10s: %s \n" "$1" "Process dead but pidfile exists"
+ else
+ printf "%10s: %s \n" "$1" "Running [${PID}]"
+ fi
+ else
+ printf "%10s: %s \n" "$1" "Not running"
+ fi
+}
+
+case "$1" in
+
+ ##############################################################################
+ # start
+ #
+ start)
+ printf "%-50s \n" "Starting $NAME..."
+ for sensor in "${SENSORS[@]}"; do
+ start $sensor
+ done
+ ;;
+
+ ##############################################################################
+ # status
+ #
+ status)
+ printf "%-50s \n" "Checking $NAME..."
+ for sensor in "${SENSORS[@]}"; do
+ status $sensor
+ done
+ ;;
+
+ ##############################################################################
+ # stop
+ #
+ stop)
+ printf "%-50s \n" "Stopping $NAME..."
+ for sensor in "${SENSORS[@]}"; do
+ stop $sensor
+ done
+ ;;
+
+ ##############################################################################
+ # restart
+ #
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
new file mode 100644
index 0000000..979de3d
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-bro-stub
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# simulates the behavior of a sensor by sending canned telemetry data
+# to a Kafka topic.
+#
+# a subset of the canned data is randomly selected and is sent in
+# batches. the timestamp of the message is altered to match current
+# system time. the number of messages sent in each batch, along with
+# the time delay between batches can be configured.
+#
+# start-bro-stub <DELAY> <COUNT>
+#
+
+#
+# how long to delay between each 'batch' in seconds.
+#
+DELAY=${1:-{{ sensor_stubs_delay }}}
+
+#
+# how many messages to send in each 'batch'. the messages are drawn randomly
+# from the entire set of canned data.
+#
+COUNT=${2:-{{ sensor_stubs_count }}}
+
+INPUT="{{ sensor_stubs_data }}/bro.out"
+PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
+TOPIC="bro"
+
+while true; do
+
+ # transform the bro timestamp and push to kafka
+ SEARCH="\"ts\"\:[0-9]\+\."
+ REPLACE="\"ts\"\:`date +%s`\."
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC
+
+ sleep $DELAY
+done
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
new file mode 100644
index 0000000..3123782
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-snort-stub
@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# simulates the behavior of a sensor by sending canned telemetry data
+# to a Kafka topic.
+#
+# a subset of the canned data is randomly selected and is sent in
+# batches. the timestamp of the message is altered to match current
+# system time. the number of messages sent in each batch, along with
+# the time delay between batches can be configured.
+#
+# start-snort-stub <DELAY> <COUNT>
+#
+
+#
+# how long to delay between each 'batch' in seconds.
+#
+DELAY=${1:-{{ sensor_stubs_delay }}}
+
+#
+# how many messages to send in each 'batch'. the messages are drawn randomly
+# from the entire set of canned data.
+#
+COUNT=${2:-{{ sensor_stubs_count }}}
+
+INPUT="{{ sensor_stubs_data }}/snort.out"
+PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
+TOPIC="snort"
+
+while true; do
+
+ # transform the timestamp and push to kafka
+ SEARCH="[^,]\+ ,"
+ REPLACE="`date +'%m\/%d\/%y-%H:%M:%S'`.000000 ,"
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC
+
+ sleep $DELAY
+done
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub
new file mode 100644
index 0000000..1966d39
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/start-yaf-stub
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# simulates the behavior of a sensor by sending canned telemetry data
+# to a Kafka topic.
+#
+# a subset of the canned data is randomly selected and is sent in
+# batches. the timestamp of the message is altered to match current
+# system time. the number of messages sent in each batch, along with
+# the time delay between batches can be configured.
+#
+# start-yaf-stub <DELAY> <COUNT>
+#
+
+#
+# how long to delay between each 'batch' in seconds.
+#
+DELAY=${1:-{{ sensor_stubs_delay }}}
+
+#
+# how many messages to send in each 'batch'. the messages are drawn randomly
+# from the entire set of canned data.
+#
+COUNT=${2:-{{ sensor_stubs_count }}}
+
+
+INPUT="{{ sensor_stubs_data }}/yaf.out"
+PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
+TOPIC="yaf"
+
+while true; do
+
+ # transform the timestamp and push to kafka
+ SEARCH="[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\.[0-9]\+"
+ REPLACE="`date +'%Y-%m-%d %H:%M:%S'`.000"
+ shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC
+
+ sleep $DELAY
+done
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/README.md b/metron-deployment/ansible/roles/sensor-test-mode/README.md
new file mode 100644
index 0000000..37afad8
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/README.md
@@ -0,0 +1,44 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Sensor Test Mode
+================
+
+A role that configures each of the sensors to produce the maximum amount of telemetry data. This role is useful only for testing. It can be useful to support functional, performance, and load testing of Apache Metron.
+
+The role does the following to maximize the amount of telemetry data produced by each Metron sensor.
+
+- Plays a packet capture file through a network interface to simulate live network traffic.
+- Configures [YAF](https://tools.netsa.cert.org/yaf/yaf.html) with `idle-timeout=0`. This causes a flow record to be produced for every network packet received.
+- Configures [Snort](https://www.snort.org/) to produce an alert for every network packet received.
+
+Getting Started
+---------------
+
+To enable the `sensor-test-mode` role apply the role to the `sensors` host group in your Ansible playbook.
+
+```
+- hosts: sensors
+ roles:
+ - role: sensor-test-mode
+```
+
+The role has also been added to the default `metron_install.yml` playbook so that it can be turned on/off with a property in both the local Virtualbox and the remote EC2 deployments.
+
+```
+sensor_test_mode: True
+```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/defaults/main.yml b/metron-deployment/ansible/roles/sensor-test-mode/defaults/main.yml
new file mode 100644
index 0000000..46c9750
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+pcap_replay: True
+install_yaf: True
+install_snort: True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/files/example.pcap
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/files/example.pcap b/metron-deployment/ansible/roles/sensor-test-mode/files/example.pcap
new file mode 100644
index 0000000..06594ec
Binary files /dev/null and b/metron-deployment/ansible/roles/sensor-test-mode/files/example.pcap differ
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/meta/main.yml b/metron-deployment/ansible/roles/sensor-test-mode/meta/main.yml
new file mode 100644
index 0000000..0e9e5b3
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - pcap_replay
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml b/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml
new file mode 100644
index 0000000..24ca87e
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/tasks/main.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: pcap.yml
+ when: install_pcap_replay
+
+- include: yaf.yml
+ when: install_yaf
+
+- include: snort.yml
+ when: install_snort
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/tasks/pcap.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/tasks/pcap.yml b/metron-deployment/ansible/roles/sensor-test-mode/tasks/pcap.yml
new file mode 100644
index 0000000..dda1bae
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/tasks/pcap.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# load example pcap data to replay
+#
+- name: Install example pcap file
+ copy: src=example.pcap dest={{ pcap_replay_home }}/
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/tasks/snort.yml b/metron-deployment/ansible/roles/sensor-test-mode/tasks/snort.yml
new file mode 100644
index 0000000..3b1af5f
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/tasks/snort.yml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# configure snort to alert on every packet
+#
+- name: Configure snort to use a set of test rules
+ lineinfile:
+ dest: /etc/snort/snort.conf
+ line: "include $RULE_PATH/test.rules"
+
+- name: Create a snort alert for testing that alerts on every packet
+ lineinfile:
+ dest: /etc/snort/rules/test.rules
+ line: "alert tcp any any -> any any (msg:'snort test alert'; sid:999158; )"
+ create: yes
+
+- name: Configure home network
+ lineinfile:
+ dest: /etc/snort/snort.conf
+ regexp: "^ipvar HOME_NET.*$"
+ line: "ipvar HOME_NET any"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-test-mode/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-test-mode/tasks/yaf.yml b/metron-deployment/ansible/roles/sensor-test-mode/tasks/yaf.yml
new file mode 100644
index 0000000..64354ac
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-test-mode/tasks/yaf.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# configure yaf to generate a flow record for every packet
+#
+- name: Stop running instances of yaf
+ service: name=yaf state=stopped
+ ignore_errors: yes
+
+- name: Configure yaf to generate a flow record for every network packet
+ lineinfile:
+ dest: /etc/init.d/yaf
+ regexp: "^DAEMONOPTS=\"${@:2}\"$"
+ line: "DAEMONOPTS=\"${@:2} --idle-timeout 0\""
+ backup: yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/defaults/main.yml b/metron-deployment/ansible/roles/snort/defaults/main.yml
new file mode 100644
index 0000000..c8b126b
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/defaults/main.yml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+snort_version: 2.9.8.0-1
+daq_version: 2.0.6-1
+snort_topic: snort
+snort_alert_csv_path: /var/log/snort/alert.csv
+snort_src_url: "https://snort.org/downloads/archive/snort/snort-{{ snort_version }}.src.rpm"
+snort_community_rules_url: "https://www.snort.org/downloads/community/community-rules.tar.gz"
+dag_src_url: "https://snort.org/downloads/snort/daq-{{ daq_version }}.src.rpm"
+sniff_interface: eth0
+snort_home_net: any
+snort_producer_home: /opt/snort-producer
+snort_producer_start: /opt/snort-producer/start-snort-producer.sh
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/files/snort.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/files/snort.conf b/metron-deployment/ansible/roles/snort/files/snort.conf
new file mode 100644
index 0000000..b03247a
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/files/snort.conf
@@ -0,0 +1,730 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###################################################
+# This file contains a sample snort configuration.
+# You should take the following steps to create your own custom configuration:
+#
+# 1) Set the network variables.
+# 2) Configure the decoder
+# 3) Configure the base detection engine
+# 4) Configure dynamic loaded libraries
+# 5) Configure preprocessors
+# 6) Configure output plugins
+# 7) Customize your rule set
+# 8) Customize preprocessor and decoder rule set
+# 9) Customize shared object rule set
+###################################################
+
+###################################################
+# Step #1: Set the network variables. For more information, see README.variables
+###################################################
+
+# Setup the network addresses you are protecting
+ipvar HOME_NET 10.0.0.16
+
+# Set up the external network addresses. Leave as "any" in most situations
+ipvar EXTERNAL_NET any
+
+# List of DNS servers on your network
+ipvar DNS_SERVERS $HOME_NET
+
+# List of SMTP servers on your network
+ipvar SMTP_SERVERS $HOME_NET
+
+# List of web servers on your network
+ipvar HTTP_SERVERS $HOME_NET
+
+# List of sql servers on your network
+ipvar SQL_SERVERS $HOME_NET
+
+# List of telnet servers on your network
+ipvar TELNET_SERVERS $HOME_NET
+
+# List of ssh servers on your network
+ipvar SSH_SERVERS $HOME_NET
+
+# List of ftp servers on your network
+ipvar FTP_SERVERS $HOME_NET
+
+# List of sip servers on your network
+ipvar SIP_SERVERS $HOME_NET
+
+# List of ports you run web servers on
+portvar HTTP_PORTS [36,80,81,82,83,84,85,86,87,88,89,90,311,383,555,591,593,631,801,808,818,901,972,1158,1220,1414,1533,1741,1830,1942,2231,2301,2381,2578,2809,2980,3029,3037,3057,3128,3443,3702,4000,4343,4848,5000,5117,5250,5600,5814,6080,6173,6988,7000,7001,7005,7071,7144,7145,7510,7770,7777,7778,7779,8000,8001,8008,8014,8015,8020,8028,8040,8080,8081,8082,8085,8088,8090,8118,8123,8180,8181,8182,8222,8243,8280,8300,8333,8344,8400,8443,8500,8509,8787,8800,8888,8899,8983,9000,9002,9060,9080,9090,9091,9111,9290,9443,9447,9710,9788,9999,10000,11371,12601,13014,15489,19980,29991,33300,34412,34443,34444,40007,41080,44449,50000,50002,51423,53331,55252,55555,56712]
+
+# List of ports you want to look for SHELLCODE on.
+portvar SHELLCODE_PORTS !80
+
+# List of ports you might see oracle attacks on
+portvar ORACLE_PORTS 1024:
+
+# List of ports you want to look for SSH connections on:
+portvar SSH_PORTS 22
+
+# List of ports you run ftp servers on
+portvar FTP_PORTS [21,2100,3535]
+
+# List of ports you run SIP servers on
+portvar SIP_PORTS [5060,5061,5600]
+
+# List of file data ports for file inspection
+portvar FILE_DATA_PORTS [$HTTP_PORTS,110,143]
+
+# List of GTP ports for GTP preprocessor
+portvar GTP_PORTS [2123,2152,3386]
+
+# other variables, these should not be modified
+ipvar AIM_SERVERS [64.12.24.0/23,64.12.28.0/23,64.12.161.0/24,64.12.163.0/24,64.12.200.0/24,205.188.3.0/24,205.188.5.0/24,205.188.7.0/24,205.188.9.0/24,205.188.153.0/24,205.188.179.0/24,205.188.248.0/24]
+
+# Path to your rules files (this can be a relative path)
+# Note for Windows users: You are advised to make this an absolute path,
+# such as: c:\snort\rules
+var RULE_PATH rules
+var SO_RULE_PATH so_rules
+var PREPROC_RULE_PATH preproc_rules
+
+# If you are using reputation preprocessor set these
+var WHITE_LIST_PATH /etc/snort/rules
+var BLACK_LIST_PATH /etc/snort/rules
+
+###################################################
+# Step #2: Configure the decoder. For more information, see README.decode
+###################################################
+
+# Configure Snort to shows year in timestamps
+config show_year
+
+# Configure Snort to output timestamps in UTC
+config utc
+
+# Stop generic decode events:
+config disable_decode_alerts
+
+# Stop Alerts on experimental TCP options
+config disable_tcpopt_experimental_alerts
+
+# Stop Alerts on obsolete TCP options
+config disable_tcpopt_obsolete_alerts
+
+# Stop Alerts on T/TCP alerts
+config disable_tcpopt_ttcp_alerts
+
+# Stop Alerts on all other TCPOption type events:
+config disable_tcpopt_alerts
+
+# Stop Alerts on invalid ip options
+config disable_ipopt_alerts
+
+# Alert if value in length field (IP, TCP, UDP) is greater th elength of the packet
+# config enable_decode_oversized_alerts
+
+# Same as above, but drop packet if in Inline mode (requires enable_decode_oversized_alerts)
+# config enable_decode_oversized_drops
+
+# Configure IP / TCP checksum mode
+config checksum_mode: all
+
+# Configure maximum number of flowbit references. For more information, see README.flowbits
+# config flowbits_size: 64
+
+# Configure ports to ignore
+# config ignore_ports: tcp 21 6667:6671 1356
+# config ignore_ports: udp 1:17 53
+
+# Configure active response for non inline operation. For more information, see README.active
+# config response: eth0 attempts 2
+
+# Configure DAQ related options for inline operation. For more information, see README.daq
+#
+# config daq: <type>
+# config daq_dir: <dir>
+# config daq_mode: <mode>
+# config daq_var: <var>
+#
+# <type> ::= pcap | afpacket | dump | nfq | ipq | ipfw
+# <mode> ::= read-file | passive | inline
+# <var> ::= arbitrary <name>=<value passed to DAQ
+# <dir> ::= path as to where to look for DAQ module so's
+
+# Configure specific UID and GID to run snort as after dropping privs. For more information see snort -h command line options
+#
+# config set_gid:
+# config set_uid:
+
+# Configure default snaplen. Snort defaults to MTU of in use interface. For more information see README
+#
+# config snaplen:
+#
+
+# Configure default bpf_file to use for filtering what traffic reaches snort. For more information see snort -h command line options (-F)
+#
+# config bpf_file:
+#
+
+# Configure default log directory for snort to log to. For more information see snort -h command line options (-l)
+#
+# config logdir:
+
+
+###################################################
+# Step #3: Configure the base detection engine. For more information, see README.decode
+###################################################
+
+# Configure PCRE match limitations
+config pcre_match_limit: 3500
+config pcre_match_limit_recursion: 1500
+
+# Configure the detection engine See the Snort Manual, Configuring Snort - Includes - Config
+config detection: search-method ac-split search-optimize max-pattern-len 20
+
+# Configure the event queue. For more information, see README.event_queue
+config event_queue: max_queue 8 log 5 order_events content_length
+
+###################################################
+## Configure GTP if it is to be used.
+## For more information, see README.GTP
+####################################################
+
+# config enable_gtp
+
+###################################################
+# Per packet and rule latency enforcement
+# For more information see README.ppm
+###################################################
+
+# Per Packet latency configuration
+#config ppm: max-pkt-time 250, \
+# fastpath-expensive-packets, \
+# pkt-log
+
+# Per Rule latency configuration
+#config ppm: max-rule-time 200, \
+# threshold 3, \
+# suspend-expensive-rules, \
+# suspend-timeout 20, \
+# rule-log alert
+
+###################################################
+# Configure Perf Profiling for debugging
+# For more information see README.PerfProfiling
+###################################################
+
+#config profile_rules: print all, sort avg_ticks
+#config profile_preprocs: print all, sort avg_ticks
+
+###################################################
+# Configure protocol aware flushing
+# For more information see README.stream5
+###################################################
+config paf_max: 16000
+
+###################################################
+# Step #4: Configure dynamic loaded libraries.
+# For more information, see Snort Manual, Configuring Snort - Dynamic Modules
+###################################################
+
+# path to dynamic preprocessor libraries
+dynamicpreprocessor directory /usr/lib64/snort-2.9.8.0_dynamicpreprocessor
+
+# path to base preprocessor engine
+dynamicengine /usr/lib64/snort-2.9.8.0_dynamicengine/libsf_engine.so
+
+# path to dynamic rules libraries
+#dynamicdetection directory /usr/local/lib/snort_dynamicrules
+
+###################################################
+# Step #5: Configure preprocessors
+# For more information, see the Snort Manual, Configuring Snort - Preprocessors
+###################################################
+
+# GTP Control Channle Preprocessor. For more information, see README.GTP
+# preprocessor gtp: ports { 2123 3386 2152 }
+
+# Inline packet normalization. For more information, see README.normalize
+# Does nothing in IDS mode
+preprocessor normalize_ip4
+preprocessor normalize_tcp: ips ecn stream
+preprocessor normalize_icmp4
+preprocessor normalize_ip6
+preprocessor normalize_icmp6
+
+# Target-based IP defragmentation. For more inforation, see README.frag3
+preprocessor frag3_global: max_frags 65536
+preprocessor frag3_engine: policy windows detect_anomalies overlap_limit 10 min_fragment_length 100 timeout 180
+
+# Target-Based stateful inspection/stream reassembly. For more inforation, see README.stream5
+preprocessor stream5_global: track_tcp yes, \
+ track_udp yes, \
+ track_icmp no, \
+ max_tcp 262144, \
+ max_udp 131072, \
+ max_active_responses 2, \
+ min_response_seconds 5
+preprocessor stream5_tcp: policy windows, detect_anomalies, require_3whs 180, \
+ overlap_limit 10, small_segments 3 bytes 150, timeout 180, \
+ ports client 21 22 23 25 42 53 70 79 109 110 111 113 119 135 136 137 139 143 \
+ 161 445 513 514 587 593 691 1433 1521 1741 2100 3306 6070 6665 6666 6667 6668 6669 \
+ 7000 8181 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779, \
+ ports both 36 80 81 82 83 84 85 86 87 88 89 90 110 311 383 443 465 563 555 591 593 631 636 801 808 818 901 972 989 992 993 994 995 1158 1220 1414 1533 1741 1830 1942 2231 2301 2381 2578 2809 2980 3029 3037 3057 3128 3443 3702 4000 4343 4848 5000 5117 5250 5600 5814 6080 6173 6988 7907 7000 7001 7005 7071 7144 7145 7510 7802 7770 7777 7778 7779 \
+ 7801 7900 7901 7902 7903 7904 7905 7906 7908 7909 7910 7911 7912 7913 7914 7915 7916 \
+ 7917 7918 7919 7920 8000 8001 8008 8014 8015 8020 8028 8040 8080 8081 8082 8085 8088 8090 8118 8123 8180 8181 8182 8222 8243 8280 8300 8333 8344 8400 8443 8500 8509 8787 8800 8888 8899 8983 9000 9002 9060 9080 9090 9091 9111 9290 9443 9447 9710 9788 9999 10000 11371 12601 13014 15489 19980 29991 33300 34412 34443 34444 40007 41080 44449 50000 50002 51423 53331 55252 55555 56712
+preprocessor stream5_udp: timeout 180
+
+# performance statistics. For more information, see the Snort Manual, Configuring Snort - Preprocessors - Performance Monitor
+# preprocessor perfmonitor: time 300 file /var/snort/snort.stats pktcnt 10000
+
+# HTTP normalization and anomaly detection. For more information, see README.http_inspect
+preprocessor http_inspect: global iis_unicode_map unicode.map 1252 compress_depth 65535 decompress_depth 65535
+preprocessor http_inspect_server: server default \
+ http_methods { GET POST PUT SEARCH MKCOL COPY MOVE LOCK UNLOCK NOTIFY POLL BCOPY BDELETE BMOVE LINK UNLINK OPTIONS HEAD DELETE TRACE TRACK CONNECT SOURCE SUBSCRIBE UNSUBSCRIBE PROPFIND PROPPATCH BPROPFIND BPROPPATCH RPC_CONNECT PROXY_SUCCESS BITS_POST CCM_POST SMS_POST RPC_IN_DATA RPC_OUT_DATA RPC_ECHO_DATA } \
+ chunk_length 500000 \
+ server_flow_depth 0 \
+ client_flow_depth 0 \
+ post_depth 65495 \
+ oversize_dir_length 500 \
+ max_header_length 750 \
+ max_headers 100 \
+ max_spaces 200 \
+ small_chunk_length { 10 5 } \
+ ports { 36 80 81 82 83 84 85 86 87 88 89 90 311 383 555 591 593 631 801 808 818 901 972 1158 1220 1414 1533 1741 1830 1942 2231 2301 2381 2578 2809 2980 3029 3037 3057 3128 3443 3702 4000 4343 4848 5000 5117 5250 5600 5814 6080 6173 6988 7000 7001 7005 7071 7144 7145 7510 7770 7777 7778 7779 8000 8001 8008 8014 8015 8020 8028 8040 8080 8081 8082 8085 8088 8090 8118 8123 8180 8181 8182 8222 8243 8280 8300 8333 8344 8400 8443 8500 8509 8787 8800 8888 8899 8983 9000 9002 9060 9080 9090 9091 9111 9290 9443 9447 9710 9788 9999 10000 11371 12601 13014 15489 19980 29991 33300 34412 34443 34444 40007 41080 44449 50000 50002 51423 53331 55252 55555 56712 } \
+ non_rfc_char { 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 } \
+ enable_cookie \
+ extended_response_inspection \
+ inspect_gzip \
+ normalize_utf \
+ unlimited_decompress \
+ normalize_javascript \
+ apache_whitespace no \
+ ascii no \
+ bare_byte no \
+ directory no \
+ double_decode no \
+ iis_backslash no \
+ iis_delimiter no \
+ iis_unicode no \
+ multi_slash no \
+ utf_8 no \
+ u_encode yes \
+ webroot no
+
+# ONC-RPC normalization and anomaly detection. For more information, see the Snort Manual, Configuring Snort - Preprocessors - RPC Decode
+preprocessor rpc_decode: 111 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779 no_alert_multiple_requests no_alert_large_fragments no_alert_incomplete
+
+# Back Orifice detection.
+preprocessor bo
+
+# FTP / Telnet normalization and anomaly detection. For more information, see README.ftptelnet
+preprocessor ftp_telnet: global inspection_type stateful encrypted_traffic no check_encrypted
+preprocessor ftp_telnet_protocol: telnet \
+ ayt_attack_thresh 20 \
+ normalize ports { 23 } \
+ detect_anomalies
+preprocessor ftp_telnet_protocol: ftp server default \
+ def_max_param_len 100 \
+ ports { 21 2100 3535 } \
+ telnet_cmds yes \
+ ignore_telnet_erase_cmds yes \
+ ftp_cmds { ABOR ACCT ADAT ALLO APPE AUTH CCC CDUP } \
+ ftp_cmds { CEL CLNT CMD CONF CWD DELE ENC EPRT } \
+ ftp_cmds { EPSV ESTA ESTP FEAT HELP LANG LIST LPRT } \
+ ftp_cmds { LPSV MACB MAIL MDTM MIC MKD MLSD MLST } \
+ ftp_cmds { MODE NLST NOOP OPTS PASS PASV PBSZ PORT } \
+ ftp_cmds { PROT PWD QUIT REIN REST RETR RMD RNFR } \
+ ftp_cmds { RNTO SDUP SITE SIZE SMNT STAT STOR STOU } \
+ ftp_cmds { STRU SYST TEST TYPE USER XCUP XCRC XCWD } \
+ ftp_cmds { XMAS XMD5 XMKD XPWD XRCP XRMD XRSQ XSEM } \
+ ftp_cmds { XSEN XSHA1 XSHA256 } \
+ alt_max_param_len 0 { ABOR CCC CDUP ESTA FEAT LPSV NOOP PASV PWD QUIT REIN STOU SYST XCUP XPWD } \
+ alt_max_param_len 200 { ALLO APPE CMD HELP NLST RETR RNFR STOR STOU XMKD } \
+ alt_max_param_len 256 { CWD RNTO } \
+ alt_max_param_len 400 { PORT } \
+ alt_max_param_len 512 { SIZE } \
+ chk_str_fmt { ACCT ADAT ALLO APPE AUTH CEL CLNT CMD } \
+ chk_str_fmt { CONF CWD DELE ENC EPRT EPSV ESTP HELP } \
+ chk_str_fmt { LANG LIST LPRT MACB MAIL MDTM MIC MKD } \
+ chk_str_fmt { MLSD MLST MODE NLST OPTS PASS PBSZ PORT } \
+ chk_str_fmt { PROT REST RETR RMD RNFR RNTO SDUP SITE } \
+ chk_str_fmt { SIZE SMNT STAT STOR STRU TEST TYPE USER } \
+ chk_str_fmt { XCRC XCWD XMAS XMD5 XMKD XRCP XRMD XRSQ } \
+ chk_str_fmt { XSEM XSEN XSHA1 XSHA256 } \
+ cmd_validity ALLO < int [ char R int ] > \
+ cmd_validity EPSV < [ { char 12 | char A char L char L } ] > \
+ cmd_validity MACB < string > \
+ cmd_validity MDTM < [ date nnnnnnnnnnnnnn[.n[n[n]]] ] string > \
+ cmd_validity MODE < char ASBCZ > \
+ cmd_validity PORT < host_port > \
+ cmd_validity PROT < char CSEP > \
+ cmd_validity STRU < char FRPO [ string ] > \
+ cmd_validity TYPE < { char AE [ char NTC ] | char I | char L [ number ] } >
+preprocessor ftp_telnet_protocol: ftp client default \
+ max_resp_len 256 \
+ bounce yes \
+ ignore_telnet_erase_cmds yes \
+ telnet_cmds yes
+
+
+# SMTP normalization and anomaly detection. For more information, see README.SMTP
+preprocessor smtp: ports { 25 465 587 691 } \
+ inspection_type stateful \
+ b64_decode_depth 0 \
+ qp_decode_depth 0 \
+ bitenc_decode_depth 0 \
+ uu_decode_depth 0 \
+ log_mailfrom \
+ log_rcptto \
+ log_filename \
+ log_email_hdrs \
+ normalize cmds \
+ normalize_cmds { ATRN AUTH BDAT CHUNKING DATA DEBUG EHLO EMAL ESAM ESND ESOM ETRN EVFY } \
+ normalize_cmds { EXPN HELO HELP IDENT MAIL NOOP ONEX QUEU QUIT RCPT RSET SAML SEND SOML } \
+ normalize_cmds { STARTTLS TICK TIME TURN TURNME VERB VRFY X-ADAT X-DRCP X-ERCP X-EXCH50 } \
+ normalize_cmds { X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
+ max_command_line_len 512 \
+ max_header_line_len 1000 \
+ max_response_line_len 512 \
+ alt_max_command_line_len 260 { MAIL } \
+ alt_max_command_line_len 300 { RCPT } \
+ alt_max_command_line_len 500 { HELP HELO ETRN EHLO } \
+ alt_max_command_line_len 255 { EXPN VRFY ATRN SIZE BDAT DEBUG EMAL ESAM ESND ESOM EVFY IDENT NOOP RSET } \
+ alt_max_command_line_len 246 { SEND SAML SOML AUTH TURN ETRN DATA RSET QUIT ONEX QUEU STARTTLS TICK TIME TURNME VERB X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
+ valid_cmds { ATRN AUTH BDAT CHUNKING DATA DEBUG EHLO EMAL ESAM ESND ESOM ETRN EVFY } \
+ valid_cmds { EXPN HELO HELP IDENT MAIL NOOP ONEX QUEU QUIT RCPT RSET SAML SEND SOML } \
+ valid_cmds { STARTTLS TICK TIME TURN TURNME VERB VRFY X-ADAT X-DRCP X-ERCP X-EXCH50 } \
+ valid_cmds { X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
+ xlink2state { enabled }
+
+# Portscan detection. For more information, see README.sfportscan
+preprocessor sfportscan: proto { all } memcap { 10000000 } sense_level { low }
+
+# ARP spoof detection. For more information, see the Snort Manual - Configuring Snort - Preprocessors - ARP Spoof Preprocessor
+# preprocessor arpspoof
+# preprocessor arpspoof_detect_host: 192.168.40.1 f0:0f:00:f0:0f:00
+
+# SSH anomaly detection. For more information, see README.ssh
+preprocessor ssh: server_ports { 22 } \
+ autodetect \
+ max_client_bytes 19600 \
+ max_encrypted_packets 20 \
+ max_server_version_len 100 \
+ enable_respoverflow enable_ssh1crc32 \
+ enable_srvoverflow enable_protomismatch
+
+# SMB / DCE-RPC normalization and anomaly detection. For more information, see README.dcerpc2
+preprocessor dcerpc2: memcap 102400, events [co ]
+preprocessor dcerpc2_server: default, policy WinXP, \
+ detect [smb [139,445], tcp 135, udp 135, rpc-over-http-server 593], \
+ autodetect [tcp 1025:, udp 1025:, rpc-over-http-server 1025:], \
+ smb_max_chain 3, smb_invalid_shares ["C$", "D$", "ADMIN$"]
+
+# DNS anomaly detection. For more information, see README.dns
+preprocessor dns: ports { 53 } enable_rdata_overflow
+
+# SSL anomaly detection and traffic bypass. For more information, see README.ssl
+preprocessor ssl: ports { 443 465 563 636 989 992 993 994 995 5061 7801 7802 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 }, trustservers, noinspect_encrypted
+
+# SDF sensitive data preprocessor. For more information see README.sensitive_data
+preprocessor sensitive_data: alert_threshold 25
+
+# SIP Session Initiation Protocol preprocessor. For more information see README.sip
+preprocessor sip: max_sessions 40000, \
+ ports { 5060 5061 5600 }, \
+ methods { invite \
+ cancel \
+ ack \
+ bye \
+ register \
+ options \
+ refer \
+ subscribe \
+ update \
+ join \
+ info \
+ message \
+ notify \
+ benotify \
+ do \
+ qauth \
+ sprack \
+ publish \
+ service \
+ unsubscribe \
+ prack }, \
+ max_uri_len 512, \
+ max_call_id_len 80, \
+ max_requestName_len 20, \
+ max_from_len 256, \
+ max_to_len 256, \
+ max_via_len 1024, \
+ max_contact_len 512, \
+ max_content_len 2048
+
+# IMAP preprocessor. For more information see README.imap
+preprocessor imap: \
+ ports { 143 } \
+ b64_decode_depth 0 \
+ qp_decode_depth 0 \
+ bitenc_decode_depth 0 \
+ uu_decode_depth 0
+
+# POP preprocessor. For more information see README.pop
+preprocessor pop: \
+ ports { 110 } \
+ b64_decode_depth 0 \
+ qp_decode_depth 0 \
+ bitenc_decode_depth 0 \
+ uu_decode_depth 0
+
+# Modbus preprocessor. For more information see README.modbus
+preprocessor modbus: ports { 502 }
+
+# DNP3 preprocessor. For more information see README.dnp3
+preprocessor dnp3: ports { 20000 } \
+ memcap 262144 \
+ check_crc
+
+# Reputation preprocessor. For more information see README.reputation
+preprocessor reputation: \
+ memcap 500, \
+ priority whitelist, \
+ nested_ip inner, \
+ whitelist $WHITE_LIST_PATH/white_list.rules, \
+ blacklist $BLACK_LIST_PATH/black_list.rules
+
+###################################################
+# Step #6: Configure output plugins
+# For more information, see Snort Manual, Configuring Snort - Output Modules
+###################################################
+
+# unified2
+# Recommended for most installs
+# output unified2: filename merged.log, limit 128, nostamp, mpls_event_types, vlan_event_types
+
+# Additional configuration for specific types of installs
+# output alert_unified2: filename snort.alert, limit 128, nostamp
+# output log_unified2: filename snort.log, limit 128, nostamp
+
+# syslog
+# output alert_syslog: LOG_AUTH LOG_ALERT
+
+# pcap
+# output log_tcpdump: tcpdump.log
+
+# metadata reference data. do not modify these lines
+include classification.config
+include reference.config
+
+
+###################################################
+# Step #7: Customize your rule set
+# For more information, see Snort Manual, Writing Snort Rules
+#
+# NOTE: All categories are enabled in this conf file
+###################################################
+
+include $RULE_PATH/community.rules
+
+# site specific rules
+# include $RULE_PATH/local.rules
+# include $RULE_PATH/app-detect.rules
+# include $RULE_PATH/attack-responses.rules
+# include $RULE_PATH/backdoor.rules
+# include $RULE_PATH/bad-traffic.rules
+# include $RULE_PATH/blacklist.rules
+# include $RULE_PATH/botnet-cnc.rules
+# include $RULE_PATH/browser-chrome.rules
+# include $RULE_PATH/browser-firefox.rules
+# include $RULE_PATH/browser-ie.rules
+# include $RULE_PATH/browser-other.rules
+# include $RULE_PATH/browser-plugins.rules
+# include $RULE_PATH/browser-webkit.rules
+# include $RULE_PATH/chat.rules
+# include $RULE_PATH/content-replace.rules
+# include $RULE_PATH/ddos.rules
+# include $RULE_PATH/dns.rules
+# include $RULE_PATH/dos.rules
+# include $RULE_PATH/experimental.rules
+# include $RULE_PATH/exploit-kit.rules
+# include $RULE_PATH/exploit.rules
+# include $RULE_PATH/file-executable.rules
+# include $RULE_PATH/file-flash.rules
+# include $RULE_PATH/file-identify.rules
+# include $RULE_PATH/file-image.rules
+# include $RULE_PATH/file-java.rules
+# include $RULE_PATH/file-multimedia.rules
+# include $RULE_PATH/file-office.rules
+# include $RULE_PATH/file-other.rules
+# include $RULE_PATH/file-pdf.rules
+# include $RULE_PATH/finger.rules
+# include $RULE_PATH/ftp.rules
+# include $RULE_PATH/icmp-info.rules
+# include $RULE_PATH/icmp.rules
+# include $RULE_PATH/imap.rules
+# include $RULE_PATH/indicator-compromise.rules
+# include $RULE_PATH/indicator-obfuscation.rules
+# include $RULE_PATH/indicator-scan.rules
+# include $RULE_PATH/indicator-shellcode.rules
+# include $RULE_PATH/info.rules
+# include $RULE_PATH/malware-backdoor.rules
+# include $RULE_PATH/malware-cnc.rules
+# include $RULE_PATH/malware-other.rules
+# include $RULE_PATH/malware-tools.rules
+# include $RULE_PATH/misc.rules
+# include $RULE_PATH/multimedia.rules
+# include $RULE_PATH/mysql.rules
+# include $RULE_PATH/netbios.rules
+# include $RULE_PATH/nntp.rules
+# include $RULE_PATH/oracle.rules
+# include $RULE_PATH/os-linux.rules
+# include $RULE_PATH/os-mobile.rules
+# include $RULE_PATH/os-other.rules
+# include $RULE_PATH/os-solaris.rules
+# include $RULE_PATH/os-windows.rules
+# include $RULE_PATH/other-ids.rules
+# include $RULE_PATH/p2p.rules
+# include $RULE_PATH/phishing-spam.rules
+# include $RULE_PATH/policy-multimedia.rules
+# include $RULE_PATH/policy-other.rules
+# include $RULE_PATH/policy.rules
+# include $RULE_PATH/policy-social.rules
+# include $RULE_PATH/policy-spam.rules
+# include $RULE_PATH/pop2.rules
+# include $RULE_PATH/pop3.rules
+# include $RULE_PATH/protocol-dns.rules
+# include $RULE_PATH/protocol-finger.rules
+# include $RULE_PATH/protocol-ftp.rules
+# include $RULE_PATH/protocol-icmp.rules
+# include $RULE_PATH/protocol-imap.rules
+# include $RULE_PATH/protocol-nntp.rules
+# include $RULE_PATH/protocol-other.rules
+# include $RULE_PATH/protocol-pop.rules
+# include $RULE_PATH/protocol-rpc.rules
+# include $RULE_PATH/protocol-scada.rules
+# include $RULE_PATH/protocol-services.rules
+# include $RULE_PATH/protocol-snmp.rules
+# include $RULE_PATH/protocol-telnet.rules
+# include $RULE_PATH/protocol-tftp.rules
+# include $RULE_PATH/protocol-voip.rules
+# include $RULE_PATH/pua-adware.rules
+# include $RULE_PATH/pua-other.rules
+# include $RULE_PATH/pua-p2p.rules
+# include $RULE_PATH/pua-toolbars.rules
+# include $RULE_PATH/rpc.rules
+# include $RULE_PATH/rservices.rules
+# include $RULE_PATH/scada.rules
+# include $RULE_PATH/scan.rules
+# include $RULE_PATH/server-apache.rules
+# include $RULE_PATH/server-iis.rules
+# include $RULE_PATH/server-mail.rules
+# include $RULE_PATH/server-mssql.rules
+# include $RULE_PATH/server-oracle.rules
+# include $RULE_PATH/server-other.rules
+# include $RULE_PATH/server-samba.rules
+# include $RULE_PATH/server-webapp.rules
+# include $RULE_PATH/shellcode.rules
+# include $RULE_PATH/smtp.rules
+# include $RULE_PATH/snmp.rules
+# include $RULE_PATH/specific-threats.rules
+# include $RULE_PATH/spyware-put.rules
+# include $RULE_PATH/sql.rules
+# include $RULE_PATH/telnet.rules
+# include $RULE_PATH/tftp.rules
+# include $RULE_PATH/virus.rules
+# include $RULE_PATH/voip.rules
+# include $RULE_PATH/web-activex.rules
+# include $RULE_PATH/web-attacks.rules
+# include $RULE_PATH/web-cgi.rules
+# include $RULE_PATH/web-client.rules
+# include $RULE_PATH/web-coldfusion.rules
+# include $RULE_PATH/web-frontpage.rules
+# include $RULE_PATH/web-iis.rules
+# include $RULE_PATH/web-misc.rules
+# include $RULE_PATH/web-php.rules
+# include $RULE_PATH/x11.rules
+
+###################################################
+# Step #8: Customize your preprocessor and decoder alerts
+# For more information, see README.decoder_preproc_rules
+###################################################
+
+# decoder and preprocessor event rules
+# include $PREPROC_RULE_PATH/preprocessor.rules
+# include $PREPROC_RULE_PATH/decoder.rules
+# include $PREPROC_RULE_PATH/sensitive-data.rules
+
+###################################################
+# Step #9: Customize your Shared Object Snort Rules
+# For more information, see http://vrt-blog.snort.org/2009/01/using-vrt-certified-shared-object-rules.html
+###################################################
+
+# dynamic library rules
+# include $SO_RULE_PATH/browser-ie.rules
+# include $SO_RULE_PATH/browser-other.rules
+# include $SO_RULE_PATH/exploit-kit.rules
+# include $SO_RULE_PATH/file-flash.rules
+# include $SO_RULE_PATH/file-image.rules
+# include $SO_RULE_PATH/file-java.rules
+# include $SO_RULE_PATH/file-multimedia.rules
+# include $SO_RULE_PATH/file-office.rules
+# include $SO_RULE_PATH/file-other.rules
+# include $SO_RULE_PATH/file-pdf.rules
+# include $SO_RULE_PATH/indicator-shellcode.rules
+# include $SO_RULE_PATH/malware-cnc.rules
+# include $SO_RULE_PATH/malware-other.rules
+# include $SO_RULE_PATH/netbios.rules
+# include $SO_RULE_PATH/os-linux.rules
+# include $SO_RULE_PATH/os-other.rules
+# include $SO_RULE_PATH/os-windows.rules
+# include $SO_RULE_PATH/policy-social.rules
+# include $SO_RULE_PATH/protocol-dns.rules
+# include $SO_RULE_PATH/protocol-nntp.rules
+# include $SO_RULE_PATH/protocol-other.rules
+# include $SO_RULE_PATH/protocol-snmp.rules
+# include $SO_RULE_PATH/protocol-voip.rules
+# include $SO_RULE_PATH/pua-p2p.rules
+# include $SO_RULE_PATH/server-apache.rules
+# include $SO_RULE_PATH/server-iis.rules
+# include $SO_RULE_PATH/server-mail.rules
+# include $SO_RULE_PATH/server-oracle.rules
+# include $SO_RULE_PATH/server-other.rules
+# include $SO_RULE_PATH/server-webapp.rules
+
+# legacy dynamic library rule files
+# include $SO_RULE_PATH/bad-traffic.rules
+# include $SO_RULE_PATH/browser-ie.rules
+# include $SO_RULE_PATH/chat.rules
+# include $SO_RULE_PATH/dos.rules
+# include $SO_RULE_PATH/exploit.rules
+# include $SO_RULE_PATH/file-flash.rules
+# include $SO_RULE_PATH/icmp.rules
+# include $SO_RULE_PATH/imap.rules
+# include $SO_RULE_PATH/misc.rules
+# include $SO_RULE_PATH/multimedia.rules
+# include $SO_RULE_PATH/netbios.rules
+# include $SO_RULE_PATH/nntp.rules
+# include $SO_RULE_PATH/p2p.rules
+# include $SO_RULE_PATH/smtp.rules
+# include $SO_RULE_PATH/snmp.rules
+# include $SO_RULE_PATH/specific-threats.rules
+# include $SO_RULE_PATH/web-activex.rules
+# include $SO_RULE_PATH/web-client.rules
+# include $SO_RULE_PATH/web-iis.rules
+# include $SO_RULE_PATH/web-misc.rules
+
+# Event thresholding or suppression commands. See threshold.conf
+include threshold.conf
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/meta/main.yml b/metron-deployment/ansible/roles/snort/meta/main.yml
new file mode 100644
index 0000000..ebca8ed
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/meta/main.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - epel
+ - libselinux-python
+ - build-tools
+ - kafka-client
+
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/tasks/daq.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/tasks/daq.yml b/metron-deployment/ansible/roles/snort/tasks/daq.yml
new file mode 100644
index 0000000..c8bd4b0
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/tasks/daq.yml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download daq
+ get_url:
+ url: "{{ dag_src_url }}"
+ dest: "/tmp/daq-{{ daq_version }}.src.rpm"
+
+- name: Build daq
+ shell: "rpmbuild --rebuild daq-{{ daq_version }}.src.rpm"
+ args:
+ chdir: /tmp
+ creates: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+
+- name: Install daq
+ yum:
+ name: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/tasks/main.yml b/metron-deployment/ansible/roles/snort/tasks/main.yml
new file mode 100644
index 0000000..4736bee
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/tasks/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: daq.yml
+- include: snort.yml
+- include: producer.yml
+- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/tasks/nic.yml b/metron-deployment/ansible/roles/snort/tasks/nic.yml
new file mode 100644
index 0000000..6053618
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/tasks/nic.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+ shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/tasks/producer.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/tasks/producer.yml b/metron-deployment/ansible/roles/snort/tasks/producer.yml
new file mode 100644
index 0000000..999192b
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/tasks/producer.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Snort producer home directory
+ file:
+ path: "{{ snort_producer_home }}"
+ state: directory
+ mode: 0755
+
+- name: Install Snort producer start script
+ template: src=start-snort-producer.sh dest={{ snort_producer_home }}/start-snort-producer.sh mode=0755
+
+- name: Install init.d service script
+ template: src=snort-producer dest=/etc/init.d/snort-producer mode=0755
+
+- name: Start Snort producer
+ service: name=snort-producer state=restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/tasks/snort.yml b/metron-deployment/ansible/roles/snort/tasks/snort.yml
new file mode 100644
index 0000000..de26936
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/tasks/snort.yml
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download snort
+ get_url:
+ url: "{{ snort_src_url }}"
+ dest: "/tmp/snort-{{ snort_version }}.src.rpm"
+
+- name: Build snort
+ shell: "rpmbuild --rebuild snort-{{ snort_version }}.src.rpm"
+ args:
+ chdir: /tmp
+ creates: /root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
+
+- name: Install snort
+ yum:
+ name: /root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Download snort community rules
+ get_url:
+ url: "{{ snort_community_rules_url }}"
+ dest: "/tmp/community-rules.tar.gz"
+
+- name: Extract tarball
+ unarchive:
+ src: "/tmp/community-rules.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/community-rules"
+
+- name: Install snort rules
+ shell: "{{ item }}"
+ args:
+ chdir: /tmp
+ with_items:
+ - cp -r community-rules/community.rules /etc/snort/rules
+ - touch /etc/snort/rules/white_list.rules
+ - touch /etc/snort/rules/black_list.rules
+ - touch /var/log/snort/alerts
+ - chown -R snort:snort /etc/snort
+
+- name: Uncomment all snort community rules
+ shell: sed -i 's/^# alert/alert/' /etc/snort/rules/community.rules
+
+- name: Download snort configuration
+ copy: src=snort.conf dest=/etc/snort/snort.conf
+
+- name: Configure home network
+ lineinfile:
+ dest: /etc/snort/snort.conf
+ regexp: "^ipvar HOME_NET.*$"
+ line: "ipvar HOME_NET {{ snort_home_net }}"
+
+- name: Configure alerting
+ lineinfile:
+ dest: /etc/snort/snort.conf
+ line: "output alert_csv: {{ snort_alert_csv_path }} default"
+
+- name: Configure sysconfig
+ lineinfile:
+ dest: /etc/sysconfig/snort
+ regexp: "{{ item.regexp }}"
+ line: "{{ item.line }}"
+ with_items:
+ - { regexp: "^ALERTMODE=.*$", line: "ALERTMODE=" }
+ - { regexp: "^NO_PACKET_LOG=.*$", line: "NO_PACKET_LOG=1" }
+ - { regexp: "^INTERFACE=.*$", line: "INTERFACE={{ sniff_interface }}" }
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/templates/snort-producer
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/templates/snort-producer b/metron-deployment/ansible/roles/snort/templates/snort-producer
new file mode 100644
index 0000000..1cb68ca
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/templates/snort-producer
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Snort Kafka producer daemon
+# chkconfig: 345 20 80
+# description: Runs Snort Kafka producer
+# processname: snort-producer
+#
+NAME=snort-producer
+DESC="Executes Snort Kafka producer"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_PATH="{{ snort_producer_home }}"
+DAEMON="{{ snort_producer_start }}"
+DAEMONOPTS="${@:2}"
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ PGID=`ps -o pgid= $PID | xargs`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ kill -- -$PGID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/snort/templates/start-snort-producer.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/snort/templates/start-snort-producer.sh b/metron-deployment/ansible/roles/snort/templates/start-snort-producer.sh
new file mode 100644
index 0000000..5234aae
--- /dev/null
+++ b/metron-deployment/ansible/roles/snort/templates/start-snort-producer.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# a simple script that tails the Snort alert log and pipes alerts into the
+# snort Kafka topic via the Kafka console producer
+#
+tail -F {{ snort_alert_csv_path }} | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ snort_topic }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/tap_interface/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/tap_interface/defaults/main.yml b/metron-deployment/ansible/roles/tap_interface/defaults/main.yml
new file mode 100644
index 0000000..ca752b4
--- /dev/null
+++ b/metron-deployment/ansible/roles/tap_interface/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+tap_if: tap0
+tap_ip: 10.0.0.1
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/tap_interface/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/tap_interface/tasks/main.yml b/metron-deployment/ansible/roles/tap_interface/tasks/main.yml
new file mode 100644
index 0000000..1de3abe
--- /dev/null
+++ b/metron-deployment/ansible/roles/tap_interface/tasks/main.yml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install tunctl
+ yum:
+ name: tunctl
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+
+- name: Create {{ tap_if }}
+ command: tunctl -p
+
+- name: Bring up {{ tap_if }} on {{ tap_ip }}
+ command: ifconfig {{ tap_if }} {{ tap_ip }} up
+
+- name: Put {{ tap_if }} in PROMISC
+ command: ip link set {{ tap_if }} promisc on
+
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/defaults/main.yml b/metron-deployment/ansible/roles/yaf/defaults/main.yml
new file mode 100644
index 0000000..0a0394b
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/defaults/main.yml
@@ -0,0 +1,29 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+fixbuf_version: 1.7.1
+yaf_version: 2.8.0
+yaf_home: /opt/yaf
+yaf_topic: yaf
+yaf_bin: /usr/local/bin/yaf
+yafscii_bin: /usr/local/bin/yafscii
+yaf_log: /var/log/yaf.log
+kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh
+daemon_bin: /usr/local/bin/airdaemon
+yaf_start: /opt/yaf/start-yaf.sh
+yaf_args:
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/meta/main.yml b/metron-deployment/ansible/roles/yaf/meta/main.yml
new file mode 100644
index 0000000..768bcac
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/meta/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - build-tools
+ - java_jdk
+ - libselinux-python
+ - kafka-client
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/tasks/fixbuf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/tasks/fixbuf.yml b/metron-deployment/ansible/roles/yaf/tasks/fixbuf.yml
new file mode 100644
index 0000000..9cd9244
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/tasks/fixbuf.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download fixbuf
+ get_url:
+ url: "http://tools.netsa.cert.org/releases/libfixbuf-{{fixbuf_version}}.tar.gz"
+ dest: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
+
+- name: Extract fixbuf tarball
+ unarchive:
+ src: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/libfixbuf-{{fixbuf_version}}"
+
+- name: Compile and Install fixbuf
+ shell: "{{item}}"
+ args:
+ chdir: "/tmp/libfixbuf-{{fixbuf_version}}"
+ with_items:
+ - ./configure
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/tasks/main.yml b/metron-deployment/ansible/roles/yaf/tasks/main.yml
new file mode 100644
index 0000000..7d21348
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: fixbuf.yml
+- include: yaf.yml
+- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/tasks/nic.yml b/metron-deployment/ansible/roles/yaf/tasks/nic.yml
new file mode 100644
index 0000000..6053618
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/tasks/nic.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+ shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/tasks/yaf.yml b/metron-deployment/ansible/roles/yaf/tasks/yaf.yml
new file mode 100644
index 0000000..6a0c651
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/tasks/yaf.yml
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download yaf
+ get_url:
+ url: "http://tools.netsa.cert.org/releases/yaf-{{yaf_version}}.tar.gz"
+ dest: "/tmp/yaf-{{yaf_version}}.tar.gz"
+
+- name: Extract yaf tarball
+ unarchive:
+ src: "/tmp/yaf-{{yaf_version}}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: /usr/local/bin/yaf
+
+- name: Compile and install yaf
+ shell: "{{item}}"
+ args:
+ chdir: "/tmp/yaf-{{yaf_version}}"
+ creates: /usr/local/bin/yaf
+ with_items:
+ - ./configure --enable-applabel --enable-plugins
+ - make
+ - make install
+
+- name: Create yaf home directory
+ file:
+ path: "{{ yaf_home }}"
+ state: directory
+ mode: 0755
+
+- name: Install yaf start script
+ template: src=start-yaf.sh dest={{ yaf_home }}/start-yaf.sh mode=0755
+
+- name: Install init.d service script
+ template: src=yaf dest=/etc/init.d/yaf mode=0755
+
+- name: Start yaf
+ service: name=yaf state=restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/templates/start-yaf.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/templates/start-yaf.sh b/metron-deployment/ansible/roles/yaf/templates/start-yaf.sh
new file mode 100644
index 0000000..9660e72
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/templates/start-yaf.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# a very simply metron probe that captures the output of yaf - yet another
+# flowmeter - and sends the output to kafka so that it can be consumed
+# by metron
+#
+{{ yaf_bin }} --in {{ sniff_interface }} --live pcap "${@:1}" | {{ yafscii_bin }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}
[24/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ntp/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ntp/tasks/main.yml b/metron-deployment/ansible/roles/ntp/tasks/main.yml
new file mode 100644
index 0000000..16274a1
--- /dev/null
+++ b/metron-deployment/ansible/roles/ntp/tasks/main.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- set_fact:
+ ntp_package_name: ntp
+ ntp_service_name: ntpd
+ when: ansible_distribution == "CentOS"
+
+- set_fact:
+ ntp_package_name: ntp
+ ntp_service_name: ntp
+ when: ansible_distribution == "Ubuntu"
+
+- name: Install ntp
+ package:
+ name: "{{ ntp_package_name }}"
+ state: installed
+
+- name: Ensure ntp is running and enabled
+ service:
+ name: "{{ ntp_service_name }}"
+ state: started
+ enabled: yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/README.md b/metron-deployment/ansible/roles/opentaxii/README.md
new file mode 100644
index 0000000..7c111de
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/README.md
@@ -0,0 +1,178 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# OpenTAXII
+
+Installs [OpenTAXII](https://github.com/EclecticIQ/OpenTAXII) as a deamon that can be launched via a SysV service script. The complementary client implementation, [Cabby](https://github.com/EclecticIQ/cabby) is also installed.
+
+OpenTAXII is a robust Python implementation of TAXII Services that delivers a rich feature set and friendly pythonic API. [TAXII](https://stixproject.github.io/) (Trusted Automated eXchange of Indicator Information) is a collection of specifications defining a set of services and message exchanges used for sharing cyber threat intelligence information between parties.
+
+## Getting Started
+
+After deployment completes the OpenTAXII service is installed and running. A set of [Hail a TAXII](http://hailataxii.com/) threat intel collections have been defined and configured. Use the `status` option to view the collections that have been defined.
+
+```
+$ service opentaxii status
+Checking opentaxii... Running
+guest.phishtank_com 0
+guest.Abuse_ch 0
+guest.CyberCrime_Tracker 0
+guest.EmergingThreats_rules 0
+guest.Lehigh_edu 0
+guest.MalwareDomainList_Hostlist 0
+guest.blutmagie_de_torExits 0
+guest.dataForLast_7daysOnly 0
+guest.dshield_BlockList 0
+```
+
+Notice that each collections contain zero records. None of the data is automatically synced during deployment. To sync the data manually use the `sync` option as defined below. The following example does not provide a begin and end time so the data will be fetched for the current day only.
+
+```
+# service opentaxii sync guest.blutmagie_de_torExits
+2016-04-21 20:34:42,511 INFO: Starting new HTTP connection (1): localhost
+2016-04-21 20:34:42,540 INFO: Response received for Inbox_Message from http://localhost:9000/services/inbox
+2016-04-21 20:34:42,542 INFO: Sending Inbox_Message to http://localhost:9000/services/inbox
+...
+2016-04-21 20:34:42,719 INFO: Response received for Poll_Request from http://localhost:9000/services/poll
+2016-04-21 20:34:42,719 INFO: Content blocks count: 1618, is partial: False
+```
+
+The OpenTAXII service now contains 1,618 threat intel records indicating Tor Exit nodes.
+
+```
+[root@source ~]# service opentaxii status
+Checking opentaxii... Running
+guest.phishtank_com 0
+guest.Abuse_ch 0
+guest.CyberCrime_Tracker 0
+guest.EmergingThreats_rules 0
+guest.Lehigh_edu 0
+guest.MalwareDomainList_Hostlist 0
+guest.blutmagie_de_torExits 1618
+guest.dataForLast_7daysOnly 0
+guest.dshield_BlockList 0
+```
+
+## Usage
+
+A standard SysV script has been installed to manage OpenTAXII. The following functions are available.
+
+`start` `stop` `restart` the OpenTAXII service
+
+`status` of the OpenTAXII service. The command displays the collections that have been defined and the number of records in each.
+
+```
+$ service opentaxii status
+Checking opentaxii... Running
+guest.phishtank_com 984
+guest.Abuse_ch 45
+guest.CyberCrime_Tracker 482
+guest.EmergingThreats_rules 0
+guest.Lehigh_edu 1030
+guest.MalwareDomainList_Hostlist 84
+guest.blutmagie_de_torExits 3236
+guest.dataForLast_7daysOnly 3377
+guest.dshield_BlockList 0
+```
+
+`setup` Initializes the services and collections required to operate the OpenTAXII service. This will destroy all existing data. The user is prompted to continue before any data is destroyed.
+
+```
+# service opentaxii setup
+WARNING: force reset and destroy all opentaxii data? [Ny]: y
+Stopping opentaxii ..Ok
+2016-04-21T19:56:01.886157Z [opentaxii.server] info: api.persistence.loaded {timestamp=2016-04-21T19:56:01.886157Z, logger=opentaxii.server, api_class=SQLDatabaseAPI, event=api.persistence.loaded, level=info}
+2016-04-21T19:56:01.896503Z [opentaxii.server] info: api.auth.loaded {timestamp=2016-04-21T19:56:01.896503Z, logger=opentaxii.server, api_class=SQLDatabaseAPI, event=api.auth.loaded, level=info}
+2016-04-21T19:56:01.896655Z [opentaxii.server] info: taxiiserver.configured {timestamp=2016-04-21T19:56:01.896655Z, logger=opentaxii.server, event=taxiiserver.configured, level=info}
+...
+Ok
+```
+
+`sync [collection] [begin-at] [end-at]` Syncs the threat intel data available at [Hail a TAXII](http://hailataxii.com/). If no begin and end date is provided then data is synced over the current day only.
+ - `collection` Name of the collection to sync.
+ - `begin-at` Exclusive begin of time window; ISO8601
+ - `end-at` Inclusive end of time window; ISO8601
+
+```
+$ service opentaxii sync guest.phishtank_com
++ /usr/local/opentaxii/opentaxii-venv/bin/taxii-proxy --poll-path http://hailataxii.com/taxii-data --poll-collection guest.phishtank_com --inbox-path http://localhost:9000/services/guest.phishtank_com-inbox --inbox-collection guest.phishtank_com --binding urn:stix.mitre.org:xml:1.1.1 --begin 2016-04-21 --end 2016-04-22
+2016-04-21 17:36:23,778 INFO: Sending Poll_Request to http://hailataxii.com/taxii-data
+2016-04-21 17:36:23,784 INFO: Starting new HTTP connection (1): hailataxii.com
+2016-04-21 17:36:24,175 INFO: Response received for Poll_Request from http://hailataxii.com/taxii-data
+2016-04-21 17:36:24,274 INFO: Sending Inbox_Message to http://localhost:9000/services/guest.phishtank_com-inbox
+...
+2016-04-21 17:36:34,867 INFO: Response received for Poll_Request from http://localhost:9000/services/guest.phishtank_com-poll
+2016-04-21 17:36:34,868 INFO: Content blocks count: 6993, is partial: False
+```
+
+### Troubleshooting
+
+Should you need to explore the installation, here are instructions on doing so.
+
+OpenTAXII is installed in a virtual environment. Before exploring the environment run the following commands to perform the necessary setup. The specific paths may change depending on your Ansible settings.
+
+```
+export LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
+export OPENTAXII_CONFIG=/usr/local/opentaxii/etc/opentaxii-conf.yml
+cd /usr/local/opentaxii
+. opentaxii-venv/bin/activate
+```
+
+Discover available services.
+
+```
+taxii-discovery --discovery http://localhost:9000/services/discovery
+taxii-discovery --discovery http://hailataxii.com/taxii-data
+```
+
+Explore available collections.
+
+```
+taxii-collections --discovery http://localhost:9000/services/discovery
+taxii-collections --discovery http://hailataxii.com/taxii-data
+```
+
+Read data from a collection.
+
+```
+taxii-poll --discovery http://localhost:9000/services/discovery -c guest.phishtank_com
+taxii-poll --discovery http://hailataxii.com/taxii-data -c guest.phishtank_com --begin 2016-04-20
+```
+
+Manually load data into a collection.
+
+```
+taxii-push \
+ --discovery http://localhost:9000/services/discovery \
+ --dest phishtank \
+ --content-file data.xml \
+ --username guest \
+ --password guest
+```
+
+Fetch data from a remote service and mirror it locally.
+
+```
+taxii-proxy --poll-path http://hailataxii.com/taxii-data \
+ --poll-collection guest.phishtank_com \
+ --inbox-path http://localhost:9000/services/guest.phishtank_com-inbox \
+ --inbox-collection guest.phishtank_com \
+ --binding urn:stix.mitre.org:xml:1.1.1 \
+ --inbox-username guest \
+ --inbox-password guest \
+ --begin 2016-04-20
+```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/defaults/main.yml b/metron-deployment/ansible/roles/opentaxii/defaults/main.yml
new file mode 100644
index 0000000..9ab86cb
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/defaults/main.yml
@@ -0,0 +1,43 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+opentaxii_host: localhost
+opentaxii_port: 9000
+opentaxii_domain: "http://{{ opentaxii_host }}:{{ opentaxii_port }}"
+opentaxii_bind: "{{ opentaxii_host }}:{{ opentaxii_port }}"
+opentaxii_home: /usr/local/opentaxii
+opentaxii_venv: opentaxii-venv
+opentaxii_bin: "{{ opentaxii_home }}/{{ opentaxii_venv }}/bin"
+opentaxii_user: guest
+opentaxii_pass: guest
+opentaxii_workers: 2
+opentaxii_loglevel: info
+opentaxii_timeout: 300
+opentaxii_auth_db: "{{ opentaxii_home }}/data/auth.db"
+opentaxii_data_db: "{{ opentaxii_home }}/data/data.db"
+opentaxii_salt: "@#L:KJDASLKJASD@"
+python27_home: /opt/rh/python27/root
+opentaxii_available_collections:
+ - guest.phishtank_com
+ - guest.Abuse_ch
+ - guest.CyberCrime_Tracker
+ - guest.EmergingThreats_rules
+ - guest.Lehigh_edu
+ - guest.MalwareDomainList_Hostlist
+ - guest.blutmagie_de_torExits
+ - guest.dataForLast_7daysOnly
+ - guest.dshield_BlockList
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/meta/main.yml b/metron-deployment/ansible/roles/opentaxii/meta/main.yml
new file mode 100644
index 0000000..841d185
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/meta/main.yml
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/tasks/dependencies.yml b/metron-deployment/ansible/roles/opentaxii/tasks/dependencies.yml
new file mode 100644
index 0000000..3b2b38a
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/tasks/dependencies.yml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install yum repositories
+ yum: name={{ item }} update_cache=yes
+ with_items:
+ - epel-release
+ - centos-release-scl
+
+- name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - python27
+ - python27-scldevel
+ - python27-python-virtualenv
+ - libxml2-devel
+ - libxslt-devel
+ - libselinux-python
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/tasks/hailataxii.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/tasks/hailataxii.yml b/metron-deployment/ansible/roles/opentaxii/tasks/hailataxii.yml
new file mode 100644
index 0000000..1eebfe6
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/tasks/hailataxii.yml
@@ -0,0 +1,45 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Deploy service/collection definitions for hailataxii.com
+ template: src={{ item }} dest={{ opentaxii_home }}/etc mode=0400
+ with_items:
+ - services.yml
+ - collections.yml
+
+- name: Add collection definitions for hailataxii.com
+ blockinfile:
+ dest: "{{ opentaxii_home }}/etc/collections.yml"
+ marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item }}"
+ block: |
+ - name: {{ item }}
+ type: DATA_SET
+ available: true
+ accept_all_content: true
+ supported_content:
+ - urn:stix.mitre.org:xml:1.1.1
+ service_ids:
+ - inbox
+ - collection
+ - poll
+ with_items: "{{ opentaxii_available_collections }}"
+
+- name: Setup opentaxii
+ shell: /etc/init.d/opentaxii setup
+
+- name: Start opentaxii
+ service: name=opentaxii state=restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/tasks/main.yml b/metron-deployment/ansible/roles/opentaxii/tasks/main.yml
new file mode 100644
index 0000000..baa6b35
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: opentaxii.yml
+- include: hailataxii.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/tasks/opentaxii.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/tasks/opentaxii.yml b/metron-deployment/ansible/roles/opentaxii/tasks/opentaxii.yml
new file mode 100644
index 0000000..c153149
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/tasks/opentaxii.yml
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create install directory
+ file: path={{ item }} state=directory mode=0755
+ with_items:
+ - "{{ opentaxii_home }}"
+ - "{{ opentaxii_home }}/etc"
+ - "{{ opentaxii_home }}/data"
+ - "{{ opentaxii_home }}/bin"
+
+- name: Create virtual environment
+ shell: "{{ python27_home }}/usr/bin/virtualenv {{ opentaxii_venv }}"
+ args:
+ chdir: "{{ opentaxii_home }}"
+ creates: "{{ opentaxii_home }}/{{ opentaxii_venv }}"
+ environment:
+ LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
+
+- name: Install opentaxii, cabby, and gunicorn
+ shell: "{{ opentaxii_bin }}/pip install {{ item }}"
+ environment:
+ LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
+ with_items:
+ - opentaxii
+ - cabby
+ - gunicorn
+
+- name: Deploy collection status script
+ template: src=collection-status.py dest={{ opentaxii_home }}/bin mode=0755
+
+- name: Deploy opentaxii configs
+ template: src=opentaxii-conf.yml dest={{ opentaxii_home }}/etc mode=0400
+
+- name: Deploy opentaxii service script
+ template: src=opentaxii dest=/etc/init.d/opentaxii mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/templates/collection-status.py
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/templates/collection-status.py b/metron-deployment/ansible/roles/opentaxii/templates/collection-status.py
new file mode 100644
index 0000000..2d912c9
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/templates/collection-status.py
@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from cabby import create_client
+
+try:
+ # create a connection
+ client = create_client(host='{{ opentaxii_host }}', port='{{ opentaxii_port }}', discovery_path='/services/discovery')
+
+ # iterate through each defined collection
+ collections = client.get_collections(uri='{{ opentaxii_domain }}/services/collection')
+
+ for collection in collections:
+ # how many records in each collection?
+ count = client.get_content_count(collection_name=collection.name, uri='{{ opentaxii_domain }}/services/poll')
+ print "%-50s %-10d" % (collection.name, count.count)
+except:
+ print "Services not defined"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/templates/collections.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/templates/collections.yml b/metron-deployment/ansible/roles/opentaxii/templates/collections.yml
new file mode 100644
index 0000000..07a0e5a
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/templates/collections.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+collections:
+# intentionally blank - managed by ansible
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/templates/opentaxii
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/templates/opentaxii b/metron-deployment/ansible/roles/opentaxii/templates/opentaxii
new file mode 100644
index 0000000..e934e7f
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/templates/opentaxii
@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# opentaxii daemon
+# chkconfig: 345 20 80
+# description: OpenTAXII is a robust Python implementation of TAXII Service
+# processname: opentaxii
+#
+NAME=opentaxii
+DESC="OpenTAXII is a robust Python implementation of a TAXII service"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="/var/log/$NAME.log"
+EXTRA_ARGS="${@:2}"
+CONFIRM_TIMEOUT=3
+DAEMON_PATH="{{ opentaxii_home }}"
+
+export LD_LIBRARY_PATH={{ python27_home }}/usr/lib64
+export OPENTAXII_CONFIG={{ opentaxii_home }}/etc/opentaxii-conf.yml
+
+case "$1" in
+
+ ##############################################################################
+ # start
+ #
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # setup virtual environment
+ cd $DAEMON_PATH
+ . {{ opentaxii_bin }}/activate
+
+ # kick-off the daemon
+ DAEMON="{{ opentaxii_bin }}/gunicorn"
+ DAEMONOPTS="opentaxii.http:app"
+ DAEMONOPTS+=" --daemon"
+ DAEMONOPTS+=" --pid $PIDFILE"
+ DAEMONOPTS+=" --workers {{ opentaxii_workers }}"
+ DAEMONOPTS+=" --log-level {{ opentaxii_loglevel }}"
+ DAEMONOPTS+=" --log-file $LOGFILE"
+ DAEMONOPTS+=" --timeout {{ opentaxii_timeout }}"
+ DAEMONOPTS+=" --bind {{ opentaxii_bind }}"
+ DAEMONOPTS+=" --env OPENTAXII_CONFIG={{ opentaxii_home }}/etc/opentaxii-conf.yml"
+ DAEMONOPTS+=" $EXTRA_ARGS"
+ PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1`
+ printf "%s\n" "Ok"
+ ;;
+
+ ##############################################################################
+ # status
+ #
+ status)
+ printf "%-50s" "Checking $NAME..."
+ . {{ opentaxii_bin }}/activate
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ printf "%s\n" "Running"
+ {{ opentaxii_home }}/bin/collection-status.py
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ ##############################################################################
+ # stop
+ #
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ while sleep 1
+ echo -n "."
+ kill -0 $PID >/dev/null 2>&1
+ do
+ kill $PID
+ done
+
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ ##############################################################################
+ # restart
+ #
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ ##############################################################################
+ # setup
+ #
+ setup)
+
+ # if the database file already exists; prompt for confirmation
+ if [ -f "{{ opentaxii_data_db }}" ]; then
+ read -t $CONFIRM_TIMEOUT -p "WARNING: force reset and destroy all opentaxii data? [Ny]: " REPLY
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ exit 0
+ fi
+ fi
+
+ $0 stop
+ cd $DAEMON_PATH
+ rm -f {{ opentaxii_auth_db }}
+ rm -f {{ opentaxii_data_db }}
+ {{ opentaxii_bin }}/opentaxii-create-account --username {{ opentaxii_user }} --password {{ opentaxii_pass }}
+ {{ opentaxii_bin }}/opentaxii-create-services -c {{ opentaxii_home}}/etc/services.yml
+ {{ opentaxii_bin }}/opentaxii-create-collections -c {{ opentaxii_home}}/etc/collections.yml
+ printf "%s\n" "Ok"
+ ;;
+
+ ##############################################################################
+ # sync
+ #
+ sync)
+
+ # collect the arguments
+ POLL_SOURCE="http://hailataxii.com/taxii-data"
+ COLL="$2"
+ BEGIN="${3:-`date --iso-8601`}"
+ END="${4:-`date --date=tomorrow --iso-8601`}"
+
+ # validation
+ if [ -z "$COLL" ]; then
+ echo "$0 sync [COLLECTION] [BEGIN-AT] [END-AT]"
+ echo "error: missing name of collection"
+ exit 1
+ fi
+
+ # sync the data
+ set -x
+ {{ opentaxii_bin }}/taxii-proxy \
+ --poll-path $POLL_SOURCE \
+ --poll-collection $COLL \
+ --inbox-path {{ opentaxii_domain }}/services/inbox \
+ --inbox-collection $COLL \
+ --binding urn:stix.mitre.org:xml:1.1.1 \
+ --begin $BEGIN \
+ --end $END
+ set +x
+
+ # count the number of records in the local collection
+ {{ opentaxii_bin }}/taxii-poll \
+ --discovery {{ opentaxii_domain }}/services/discovery \
+ --collection $COLL \
+ --count-only
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart|setup|sync}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/templates/opentaxii-conf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/templates/opentaxii-conf.yml b/metron-deployment/ansible/roles/opentaxii/templates/opentaxii-conf.yml
new file mode 100644
index 0000000..2ce81c0
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/templates/opentaxii-conf.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+domain: "{{ opentaxii_domain }}"
+support_basic_auth: yes
+
+persistence_api:
+ class: opentaxii.persistence.sqldb.SQLDatabaseAPI
+ parameters:
+ db_connection: "sqlite:///{{ opentaxii_data_db }}"
+ create_tables: yes
+
+auth_api:
+ class: opentaxii.auth.sqldb.SQLDatabaseAPI
+ parameters:
+ db_connection: "sqlite:///{{ opentaxii_auth_db }}"
+ create_tables: yes
+ secret: "{{ opentaxii_salt }}"
+
+logging:
+ opentaxii: debug
+ root: info
+
+hooks:
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/opentaxii/templates/services.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/opentaxii/templates/services.yml b/metron-deployment/ansible/roles/opentaxii/templates/services.yml
new file mode 100644
index 0000000..61a1d42
--- /dev/null
+++ b/metron-deployment/ansible/roles/opentaxii/templates/services.yml
@@ -0,0 +1,69 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+services:
+ #
+ # discovery: used by a TAXII Client to discover available TAXII Service
+ #
+ - id: discovery
+ type: discovery
+ address: /services/discovery
+ description: Discovery service for Apache Metron
+ advertised_services:
+ - inbox
+ - collection
+ - poll
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
+
+ #
+ # inbox: used by a TAXII Client to push information to a TAXII Server
+ #
+ - id: inbox
+ type: inbox
+ address: /services/inbox
+ description: Inbox for Apache Metron
+ destination_collection_required: yes
+ accept_all_content: yes
+ supported_content:
+ - urn:stix.mitre.org:xml:1.1.1
+ authentication_required: no
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
+
+ #
+ # collection_management: used by a TAXII Client to request information about
+ # available data collections or request a subscription.
+ #
+ - id: collection
+ type: collection_management
+ address: /services/collection
+ description: Collection management service for Apache Metron
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
+
+ #
+ # poll: used by a TAXII Client to request information from a TAXII Server
+ #
+ - id: poll
+ type: poll
+ address: /services/poll
+ description: Poll service for Apache Metron
+ subscription_required: no
+ authentication_required: no
+ protocol_bindings:
+ - urn:taxii.mitre.org:protocol:http:1.0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/README.md b/metron-deployment/ansible/roles/pcap_replay/README.md
new file mode 100644
index 0000000..01475f9
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/README.md
@@ -0,0 +1,61 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Pcap Replay
+===========
+
+This project enables packet capture data to be replayed through a network interface to simulate live network traffic. This can be used to support functional, performance, and load testing of Apache Metron.
+
+Getting Started
+---------------
+
+To replay packet capture data, simply start the `pcap-replay` SysV service. To do this run the following command.
+
+```
+service pcap-replay start
+```
+
+All additional options accepted by `tcpreplay` can be passed to the service script to modify how the network data is replayed. For example, this makes it simple to control the amount and rate of data replayed during functional, performance and load testing.
+
+Example: Replay data at a rate of 10 mbps.
+
+```
+service pcap-replay start --mbps 10
+```
+
+Example: Replay data at a rate of 10 packets per second.
+
+```
+service pcap-replay start --pps 10
+```
+
+All nodes on the same subnet with their network interface set to promiscuous mode will then be able to capture the network traffic being replayed. To validate, simply run something like the following.
+
+```
+tcpdump -i eth1
+```
+
+Data
+----
+
+An example packet capture file has been installed at `/opt/pcap-replay/example.pcap`. By default, the network traffic contained within this file is continually replayed.
+
+To replay your own packet capture data, simply add any number of files containing `libpcap` formatted packet capture data to `/opt/pcap-replay`. The files must end with the `.pcap` extension. To pick up newly installed files, simply restart the service.
+
+```
+service pcap-replay restart
+```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/defaults/main.yml b/metron-deployment/ansible/roles/pcap_replay/defaults/main.yml
new file mode 100644
index 0000000..9b948a3
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+pcap_replay_interface: eth0
+pcap_replay_home: /opt/pcap-replay
+tcpreplay_version: 4.1.1
+tcpreplay_prefix: /opt
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/meta/main.yml b/metron-deployment/ansible/roles/pcap_replay/meta/main.yml
new file mode 100644
index 0000000..841d185
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/meta/main.yml
@@ -0,0 +1,17 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/tasks/dependencies.yml b/metron-deployment/ansible/roles/pcap_replay/tasks/dependencies.yml
new file mode 100644
index 0000000..effe6c4
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/tasks/dependencies.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - libpcap
+ - libpcap-devel
+ - pcre
+ - pcre-devel
+ - zlib
+ - zlib-devel
+ - glib2-devel
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml b/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml
new file mode 100644
index 0000000..bdc2a0b
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: tcpreplay.yml
+- include: service.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/tasks/service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/tasks/service.yml b/metron-deployment/ansible/roles/pcap_replay/tasks/service.yml
new file mode 100644
index 0000000..73462bf
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/tasks/service.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create pcap directory
+ file: path={{ pcap_replay_home }} state=directory mode=0755
+
+- name: Install init.d service script
+ template: src=pcap-replay dest=/etc/init.d/pcap-replay mode=0755
+
+- name: Start pcap replay
+ service: name=pcap-replay state=restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/tasks/tcpreplay.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/tasks/tcpreplay.yml b/metron-deployment/ansible/roles/pcap_replay/tasks/tcpreplay.yml
new file mode 100644
index 0000000..e24dcf1
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/tasks/tcpreplay.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download tcpreplay
+ get_url:
+ url: "https://github.com/appneta/tcpreplay/releases/download/v{{ tcpreplay_version }}/tcpreplay-{{ tcpreplay_version }}.tar.gz"
+ dest: "/tmp/tcpreplay-{{ tcpreplay_version }}.tar.gz"
+
+- name: Extract tcpreplay tarball
+ unarchive:
+ src: "/tmp/tcpreplay-{{ tcpreplay_version }}.tar.gz"
+ dest: /opt
+ copy: no
+ creates: "/opt/tcpreplay-{{ tcpreplay_version }}"
+
+- name: Compile and install tcpreplay
+ shell: "{{ item }}"
+ args:
+ chdir: "/opt/tcpreplay-{{ tcpreplay_version }}"
+ creates: "{{ tcpreplay_prefix }}/bin/tcpreplay"
+ with_items:
+ - "./configure --prefix={{ tcpreplay_prefix }}"
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pcap_replay/templates/pcap-replay
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pcap_replay/templates/pcap-replay b/metron-deployment/ansible/roles/pcap_replay/templates/pcap-replay
new file mode 100644
index 0000000..43ecc82
--- /dev/null
+++ b/metron-deployment/ansible/roles/pcap_replay/templates/pcap-replay
@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# pcap replay daemon
+# chkconfig: 345 20 80
+# description: Replays packet capture data stored in libpcap format
+# processname: pcap-replay
+#
+
+DAEMON_PATH="{{ pcap_replay_home }}"
+PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
+IFACE="{{ pcap_replay_interface }}"
+EXTRA_ARGS="${@:2}"
+NAME=pcap-replay
+DESC="Replay packet capture data"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE=/var/log/$NAME.log
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # ensure that a pcap file exists to replay
+ if [ -z "$PCAPIN" ]; then
+ printf "%s: %s\n" "Fail: No pcap files found at " $DAEMON_PATH
+ else
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
+ DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"
+ PID=`$DAEMON $DAEMONOPTS > $LOGFILE 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ while sleep 1
+ echo -n "."
+ kill -0 $PID >/dev/null 2>&1
+ do
+ kill $PID
+ done
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/defaults/main.yml b/metron-deployment/ansible/roles/pycapa/defaults/main.yml
new file mode 100644
index 0000000..edcf980
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/defaults/main.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+pycapa_home: /usr/local/pycapa
+pycapa_venv: pycapa-venv
+pycapa_bin: "{{ pycapa_home }}/{{ pycapa_venv }}/bin"
+pycapa_log: /var/log/pycapa.log
+pycapa_topic: pcap
+pycapa_sniff_interface: "{{ sniff_interface }}"
+python27_home: /opt/rh/python27/root
+
+install_pycapa_service: True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/meta/main.yml b/metron-deployment/ansible/roles/pycapa/meta/main.yml
new file mode 100644
index 0000000..a5b54b7
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/meta/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/tasks/dependencies.yml b/metron-deployment/ansible/roles/pycapa/tasks/dependencies.yml
new file mode 100644
index 0000000..2fba127
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/tasks/dependencies.yml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install yum repositories
+ yum: name={{ item }} update_cache=yes
+ with_items:
+ - epel-release
+ - centos-release-scl
+
+- name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - python27
+ - python27-scldevel
+ - python27-python-virtualenv
+ - libpcap-devel
+ - libselinux-python
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/tasks/main.yml b/metron-deployment/ansible/roles/pycapa/tasks/main.yml
new file mode 100644
index 0000000..d2367c2
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/tasks/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: pycapa.yml
+- include: pycapa-service.yml
+ when: install_pycapa_service
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/tasks/pycapa-service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/tasks/pycapa-service.yml b/metron-deployment/ansible/roles/pycapa/tasks/pycapa-service.yml
new file mode 100644
index 0000000..017cc66
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/tasks/pycapa-service.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ pycapa_sniff_interface }}
+ shell: "ip link set {{ pycapa_sniff_interface }} promisc on"
+
+- name: Install service script
+ template: src=pycapa dest=/etc/init.d/pycapa mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/tasks/pycapa.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/tasks/pycapa.yml b/metron-deployment/ansible/roles/pycapa/tasks/pycapa.yml
new file mode 100644
index 0000000..87e964e
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/tasks/pycapa.yml
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create install directory
+ file: path={{ pycapa_home }} state=directory mode=0755
+
+- name: Create virtual environment
+ shell: "{{ python27_home }}/usr/bin/virtualenv {{ pycapa_venv }}"
+ args:
+ chdir: "{{ pycapa_home }}"
+ creates: "{{ pycapa_home }}/{{ pycapa_venv }}"
+ environment:
+ LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
+
+- name: Distribute pycapa
+ copy: src=../../../metron-sensors/pycapa dest={{ pycapa_home }} mode=0755
+
+- name: Build pycapa
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ pycapa_home }}/pycapa"
+ environment:
+ LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
+ with_items:
+ - "{{ pycapa_bin }}/pip install -r requirements.txt"
+ - "{{ pycapa_bin }}/python setup.py install"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/pycapa/templates/pycapa
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/pycapa/templates/pycapa b/metron-deployment/ansible/roles/pycapa/templates/pycapa
new file mode 100644
index 0000000..ce87477
--- /dev/null
+++ b/metron-deployment/ansible/roles/pycapa/templates/pycapa
@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron pycapa service
+# chkconfig: 345 20 80
+# description: Metron Pycapa Packet Capture Daemon
+# processname: pycapa
+#
+NAME=pycapa
+DESC="Pycapa - Apache Metron Packet Capture"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="{{ pycapa_log }}"
+EXTRA_ARGS="${@:2}"
+DAEMON_PATH="{{ pycapa_home }}"
+
+export LD_LIBRARY_PATH={{ python27_home }}/usr/lib64
+
+case "$1" in
+
+ ##############################################################################
+ # start
+ #
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # setup virtual environment
+ cd $DAEMON_PATH
+ . {{ pycapa_bin }}/activate
+
+ # kick-off the daemon
+ DAEMON_PATH="{{ pycapa_home }}"
+ DAEMON="{{ pycapa_bin }}/pycapa"
+ DAEMONOPTS+=" --producer "
+ DAEMONOPTS+=" --kafka {{ kafka_broker_url }}"
+ DAEMONOPTS+=" --topic {{ pycapa_topic }}"
+ DAEMONOPTS+=" --interface {{ pycapa_sniff_interface }}"
+ DAEMONOPTS+=" $EXTRA_ARGS"
+
+ PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ ##############################################################################
+ # status
+ #
+ status)
+ printf "%-50s" "Checking $NAME..."
+ . {{ pycapa_bin }}/activate
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ printf "%s\n" "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ ##############################################################################
+ # stop
+ #
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ while sleep 1
+ echo -n "."
+ kill -0 $PID >/dev/null 2>&1
+ do
+ kill $PID
+ done
+
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ ##############################################################################
+ # restart
+ #
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/python-pip/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/python-pip/tasks/main.yml b/metron-deployment/ansible/roles/python-pip/tasks/main.yml
new file mode 100644
index 0000000..f8c0489
--- /dev/null
+++ b/metron-deployment/ansible/roles/python-pip/tasks/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install Python's pip on Centos
+ yum: name=python-pip
+ when: ansible_distribution == "CentOS"
+
+- name: Install Python's pip on Ubuntu
+ apt: name=python-pip force=yes
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/README.md b/metron-deployment/ansible/roles/sensor-stubs/README.md
new file mode 100644
index 0000000..015f1d1
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/README.md
@@ -0,0 +1,99 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Sensor Stubs
+
+A service has been created to simulate the behavior of a sensor by sending canned telemetry data to a Kafka topic. These "Sensor Stubs" consume fewer resources than the actual sensor that they replace.
+
+### (Q) How do the sensor stubs work?
+
+The stubs are installed with a set of canned data for each sensor type; Bro, Snort and YAF. A subset of this canned data is randomly selected and sent to the Kafka topic in batches. The timestamp of each message is updated to match current system time.
+
+### (Q) How do I configure the message rate?
+
+The number of telemetry messages sent in each batch, along with the time delay between batches is configurable. Before installation, these values can be configured by redefining `sensor_stubs_delay` and `sensor_stubs_count`. The values can also be configured by altering the deployed system service script at `/etc/init.d/sensor-stubs`.
+
+### (Q) How do I install the sensor stubs?
+
+Using the default playbooks, this role can be installed by using the Ansible tag `sensor-stubs`. This service is installed on the same hosts where the sensors would be; defined by the `sensors` host group.
+
+The defaults for the "Full Dev" environment have been changed so that the Sensor Stubs are installed by default, rather than the sensors themselves. The Amazon EC2 environment continues to install the original sensors by default.
+
+### (Q) How do I use the sensor stubs?
+
+Start all sensor stubs. The output includes the PID for each running sensor stub.
+```
+$ service sensor-stubs start
+Starting sensor-stubs...
+ bro: Ok [26505]
+ yaf: Ok [26507]
+ snort: Ok [26509]
+```
+
+Check the status of each sensor stub.
+```
+$ service sensor-stubs status
+Checking sensor-stubs...
+ bro: Running [26505]
+ yaf: Running [26507]
+ snort: Running [26509]
+```
+
+Stop all sensor stubs.
+```
+$ service sensor-stubs stop
+Stopping sensor-stubs...
+.. bro: Ok [26505]
+.. yaf: Ok [26507]
+.. snort: Ok [26509]
+```
+
+Check the status. All sensor stubs should be stopped.
+```
+$ service sensor-stubs status
+Checking sensor-stubs...
+ bro: Not running
+ yaf: Not running
+ snort: Not running
+```
+
+Start only the Bro sensor stub.
+```
+$ service sensor-stubs start bro
+Starting sensor-stubs...
+ bro: OK [11616]
+```
+
+Stop the Bro sensor stub.
+```
+$ service sensor-stubs stop bro
+Stopping sensor-stubs...
+.. bro: Ok [11616]
+```
+
+### (Q) How do I install the original sensors?
+
+The default behavior can be changed by skipping the `sensor-stubs` flag and including the `sensors` flag. For example, to deploy "Full Dev" with the original sensors run the following command.
+
+```
+cd metron-deployment/development/centos6
+vagrant --ansible-skip-tags="sensor-stubs,solr" up
+```
+
+### (Q) Where does the mock data come from?
+
+The data produced by the sensor stubs was generated by running the sensors against the example pcap file that is distributed with Metron. This ensures that the data produced by the sensor stubs is similar to the data produced when using the actual sensors.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/defaults/main.yml b/metron-deployment/ansible/roles/sensor-stubs/defaults/main.yml
new file mode 100644
index 0000000..e8efb9e
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/defaults/main.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+sensor_stubs_home: /opt/sensor-stubs
+sensor_stubs_bin: "{{ sensor_stubs_home }}/bin"
+sensor_stubs_data: "{{ sensor_stubs_home }}/data"
+sensor_stubs_log: /var/log/sensor-stubs.log
+
+sensor_stubs_delay: 2
+sensor_stubs_count: 10
+kafka_home: /usr/hdp/current/kafka-broker
\ No newline at end of file
[26/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
new file mode 100644
index 0000000..8c1bc33
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Setup Ambari repo on CentOS
+ get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
+
+- name: Update package cache on CentOS
+ yum: name=* update_cache=yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
new file mode 100644
index 0000000..a0ad679
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- name: Setup Ambari repo on Ubuntu
+ shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | sudo tee /etc/apt/sources.list.d/ambari.list
+
+- name: Update package cache on Ubuntu
+ apt: update_cache=yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml b/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
new file mode 100644
index 0000000..730607d
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+#
+# for CentOS...
+#
+# this is not needed for CentOS. the elasticsearch repository is automatically
+# added by the mpack. there is currently an issue that needs resolved that
+# prevents us from doing the same in Ubuntu
+#
+
+#
+# for Ubuntu...
+#
+- name: Setup Elasticsearch packages repo on Ubuntu
+ shell: echo "deb {{ ubuntu_elasticsearch_packages_repo }} stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch.list
+ when: ansible_distribution == "Ubuntu"
+
+- name: Setup Elasticsearch curator repo on Ubuntu
+ shell: echo "deb {{ ubuntu_elasticsearch_curator_repo }} stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch.list
+ when: ansible_distribution == "Ubuntu"
+
+- name: Update package cache
+ apt: update_cache=yes
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml b/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml
new file mode 100644
index 0000000..4688332
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/hostname.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# ambari uses socket.getfqdn() to find the hostname. with 'localhost.localdomain'
+# in '/etc/hosts' this function will report the hostname as 'localhost.localdomain'
+# rather than 'node1' as would be expected. other functions like socket.gethostname()
+# will always return 'node1' as expected. ambari needs to see 'node1' to be able to
+# communicate between the master and agents.
+
+- name: Remove ipv4 'localhost.localdomain' from /etc/hosts
+ lineinfile: dest=/etc/hosts state=absent regexp="^127.0.0.1(.*)localdomain(.*)$"
+
+- name: Remove ipv6 'localhost.localdomain' from /etc/hosts
+ lineinfile: dest=/etc/hosts state=absent regexp="^::1(.*)localdomain(.*)$"
+
+- name: Add localhost to /etc/hosts
+ lineinfile: dest=/etc/hosts line="127.0.0.1 localhost"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
new file mode 100644
index 0000000..27e67d5
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Stop iptables on CentOS
+ ignore_errors: yes
+ service: name=iptables state=stopped enabled=no
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
new file mode 100644
index 0000000..9fdda7e
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Disable firewall on Ubuntu
+ shell: ufw disable
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml b/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml
new file mode 100644
index 0000000..2c4a2e0
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/logrotate.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Logrotate Script for Hadoop Services
+ template:
+ src: "metron-hadoop-logrotate.yml"
+ dest: "/etc/logrotate.d/metron-ambari"
+ mode: 0644
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
new file mode 100644
index 0000000..b3258a9
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+- include: iptables-centos.yml
+ static: no
+ when: ansible_distribution == "CentOS"
+
+- include: iptables-ubuntu.yml
+ static: no
+ when: ansible_distribution == "Ubuntu"
+
+- include: hostname.yml
+
+- include: ambari-repo-centos.yml
+ static: no
+ when: ansible_distribution == "CentOS"
+
+- include: ambari-repo-ubuntu.yml
+ static: no
+ when: ansible_distribution == "Ubuntu"
+
+- include: elasticsearch-repo.yml
+- include: nodejs.yml
+- include: logrotate.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
new file mode 100644
index 0000000..0dbf9f7
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# for CentOS...
+#
+- name: Install Nodejs repo on CentOS
+ shell: curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
+ args:
+ warn: false
+ when: ansible_distribution == "CentOS"
+
+#
+# for Ubuntu...
+#
+- name: Install Nodejs repo on Ubuntu
+ shell: curl -sL https://deb.nodesource.com/setup_6.x | sudo -E bash -
+ args:
+ warn: false
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml b/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml
new file mode 100644
index 0000000..d95c10b
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_common/templates/metron-hadoop-logrotate.yml
@@ -0,0 +1,149 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Hadoop HDFS Logs
+/var/log/hadoop/hdfs/*.log* {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/hadoop/hdfs/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/hadoop/hdfs/*.audit {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+#Hadoop Yarn Logs
+/var/log/hadoop/yarn/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+#Hadoop Mapreduce Logs
+/var/log/hadoop/mapreduce/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+#Storm Logs
+/var/log/storm/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/storm/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+#Kafka Logs
+/var/log/kafka/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/kafka/*.err {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+#HBase Logs
+/var/log/hbase/*.log* {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/hbase/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/hbase/*.audit {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+#Zookeeper Logs
+/var/log/zookeeper/*.log {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
+/var/log/zookeeper/*.out {
+ {{ hadoop_logrotate_frequency }}
+ rotate {{ hadoop_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
new file mode 100644
index 0000000..e0de145
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+zookeeper_data_dir: /hadoop/zookeeper
+namenode_checkpoint_dir: /hadoop/hdfs/namesecondary
+namenode_name_dir: /hadoop/hdfs/namenode
+datanode_data_dir: /hadoop/hdfs/data
+journalnode_edits_dir: /hadoop/hdfs/journalnode
+jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs
+nodemanager_local_dirs: /hadoop/yarn/local
+timeline_ldb_store_path: /hadoop/yarn/timeline
+timeline_ldb_state_path: /hadoop/yarn/timeline
+nodemanager_log_dirs: /hadoop/yarn/log
+storm_local_dir: /hadoop/storm
+kafka_log_dirs: /kafka-log
+cluster_type: small_cluster
+nodemanager_mem_mb : 4096
+mapred_map_java_opts : -Xmx1024m
+mapred_reduce_java_opts : -Xmx1024m
+mapred_map_mem_mb : 1229
+mapred_reduce_mem_mb : 1229
+topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf'
+hdp_stack: "2.5"
+elasticsearch_network_interface: _site_
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/meta/main.yml b/metron-deployment/ansible/roles/ambari_config/meta/main.yml
new file mode 100644
index 0000000..8f65a28
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_config/meta/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - epel
+ - python-pip
+ - httplib2
+ - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
new file mode 100644
index 0000000..fdb21c7
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install python-requests on CentOS
+ yum: name=python-requests
+ when: ansible_distribution == "CentOS"
+
+- name: Install python-requests on Ubuntu
+ apt: name=python-requests force=yes
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/main.yml b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml
new file mode 100644
index 0000000..d5f38b1
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include_vars: "{{ cluster_type }}.yml"
+
+- include: dependencies.yml
+
+- name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }}
+ wait_for :
+ host: "{{ ambari_host }}"
+ port: "{{ ambari_port }}"
+ timeout: 600
+
+- name: Deploy cluster with Ambari; http://{{ ambari_host }}:{{ ambari_port }}
+ ambari_cluster_state:
+ host: "{{ ambari_host }}"
+ port: "{{ ambari_port }}"
+ username: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ cluster_name: "{{ cluster_name }}"
+ cluster_state: present
+ blueprint_name: "{{ blueprint_name }}"
+ configurations: "{{ configurations }}"
+ wait_for_complete: True
+ blueprint_var: "{{ blueprint }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
new file mode 100644
index 0000000..6a60902
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -0,0 +1,135 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# vars file for single_node_vm blueprint
+
+hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
+hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, HDFS_CLIENT, NODEMANAGER, YARN_CLIENT, MAPREDUCE2_CLIENT]
+spark_master: [SPARK_JOBHISTORYSERVER]
+spark_slave: [SPARK_CLIENT]
+storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
+storm_slave: [SUPERVISOR]
+kafka_broker: [KAFKA_BROKER]
+zookeeper_master: [ZOOKEEPER_SERVER]
+zookeeper_slave: [ZOOKEEPER_CLIENT]
+hbase_master: [HBASE_MASTER, HBASE_CLIENT]
+hbase_slave: [HBASE_REGIONSERVER]
+es_master: [ES_MASTER]
+kibana_master: [KIBANA_MASTER]
+metron_indexing: [METRON_INDEXING]
+metron_profiler: [METRON_PROFILER]
+metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
+metron_parsers : [METRON_PARSERS]
+metron_rest: [METRON_REST]
+metron_management_ui: [METRON_MANAGEMENT_UI]
+metron_alerts_ui: [METRON_ALERTS_UI]
+
+metron_components: >
+ {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
+ union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
+ union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }}
+
+cluster_name: "metron_cluster"
+blueprint_name: "metron_blueprint"
+
+configurations:
+ - zoo.cfg:
+ dataDir: '{{ zookeeper_data_dir }}'
+ - hadoop-env:
+ hadoop_heapsize: 1024
+ namenode_heapsize: 2048
+ dtnode_heapsize: 512
+ namenode_opt_permsize: 128m
+ - hbase-env:
+ hbase_regionserver_heapsize: 512
+ hbase_master_heapsize: 512
+ hbase_regionserver_xmn_max: 512
+ - hdfs-site:
+ dfs.replication: 1
+ dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}'
+ dfs.namenode.name.dir: '{{ namenode_name_dir }}'
+ dfs.datanode.data.dir: '{{ datanode_data_dir }}'
+ dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}'
+ - yarn-env:
+ nodemanager_heapsize: 512
+ yarn_heapsize: 512
+ apptimelineserver_heapsize : 512
+ resourcemanager_heapsize: 1024
+ - mapred-env:
+ jobhistory_heapsize: 256
+ - mapred-site:
+ mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}'
+ mapreduce.map.java.opts : '{{ mapred_map_java_opts }}'
+ mapreduce.reduce.java.opts : '{{ mapred_reduce_java_opts }}'
+ mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
+ mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
+ - yarn-site:
+ yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}'
+ yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}'
+ yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}'
+ yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
+ yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
+ - storm-site:
+ supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
+ storm.local.dir: '{{ storm_local_dir }}'
+ topology.classpath: '{{ topology_classpath }}'
+ - kafka-env:
+ content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
+ - kafka-broker:
+ log.dirs: '{{ kafka_log_dirs }}'
+ delete.topic.enable: "true"
+ - metron-rest-env:
+ metron_spring_profiles_active: "dev"
+ - metron-parsers-env:
+ parsers: "bro,snort"
+ - elastic-site:
+ index_number_of_shards: 1
+ index_number_of_replicas: 0
+ zen_discovery_ping_unicast_hosts: "[ {{ groups.search | join(', ') }} ]"
+ gateway_recover_after_data_nodes: 1
+ network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"
+ masters_also_are_datanodes: "1"
+
+required_configurations:
+ - metron-env:
+ storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
+ es_hosts: "{{ groups.search | join(',') }}"
+ zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
+ - metron-rest-env:
+ metron_jdbc_driver: "org.h2.Driver"
+ metron_jdbc_url: "jdbc:h2:file:~/metrondb"
+ metron_jdbc_username: "root"
+ metron_jdbc_password: "root"
+ metron_jdbc_platform: "h2"
+ - kibana-env:
+ kibana_pid_dir: /var/run/kibana
+ kibana_es_url: http://{{ groups.search[0] }}:9200
+ kibana_log_dir: /var/log/kibana
+ kibana_server_port: 5000
+ kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
+ kibana_server_host: 0.0.0.0
+
+blueprint:
+ stack_name: HDP
+ stack_version: "{{ hdp_stack }}"
+ required_configurations: "{{ required_configurations }}"
+ groups:
+ - name : host_group_1
+ cardinality: 1
+ configurations: []
+ components: "{{ metron_components }}"
+ hosts: "{{ hdp_host_group }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
new file mode 100644
index 0000000..4ec8458
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
@@ -0,0 +1,149 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
+app_timeline_server: [APP_TIMELINE_SERVER]
+hadoop_slave: [DATANODE, NODEMANAGER]
+spark_master: [SPARK_JOBHISTORYSERVER]
+storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
+storm_slave: [SUPERVISOR]
+kafka_broker: [KAFKA_BROKER]
+zookeeper_master: [ZOOKEEPER_SERVER]
+hbase_master: [HBASE_MASTER]
+hbase_slave: [HBASE_REGIONSERVER]
+hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
+es_master: [ES_MASTER]
+es_slave: [ES_SLAVE]
+kibana_master: [KIBANA_MASTER]
+metron_indexing: [METRON_INDEXING]
+metron_profiler: [METRON_PROFILER]
+metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
+metron_parsers : [METRON_PARSERS]
+metron_rest: [METRON_REST]
+metron_management_ui: [METRON_MANAGEMENT_UI]
+metron_alerts_ui: [METRON_ALERTS_UI]
+
+master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}"
+master_1_host:
+ - "{{groups.ambari_slave[0]}}"
+master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}"
+master_2_host:
+ - "{{groups.ambari_slave[1]}}"
+metron_components: >
+ {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
+ union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
+metron_host:
+ - "{{ groups.metron[0] }}"
+web_components: "{{ kibana_master | union(es_master) }}"
+web_host:
+ - "{{ groups.web[0] }}"
+slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"
+
+cluster_name: "metron"
+blueprint_name: "metron_blueprint"
+
+configurations:
+ - zoo.cfg:
+ dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
+ - hadoop-env:
+ namenode_heapsize: 2048
+ dtnode_heapsize: 1024
+ - hbase-env:
+ hbase_regionserver_heapsize: 1024
+ hbase_master_heapsize: 1024
+ - hdfs-site:
+ dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
+ dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
+ dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
+ dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
+ - mapred-site:
+ mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
+ mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
+ mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
+ - yarn-site:
+ yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
+ yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
+ yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
+ yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
+ yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
+ - storm-site:
+ supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
+ storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
+ topology.classpath: '{{ topology_classpath }}'
+ - kafka-broker:
+ log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
+ - metron-rest-env:
+ metron_spring_profiles_active: "dev"
+ - metron-env:
+ parsers: "bro,snort,yaf"
+ - elastic-site:
+ index_number_of_shards: 2
+ index_number_of_replicas: 1
+ zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]"
+ gateway_recover_after_data_nodes: 1
+ network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"
+
+required_configurations:
+ - metron-env:
+ storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
+ es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
+ zeppelin_server_url: "{{ groups.zeppelin[0] }}"
+ - metron-rest-env:
+ metron_jdbc_driver: "org.h2.Driver"
+ metron_jdbc_url: "jdbc:h2:file:~/metrondb"
+ metron_jdbc_username: "root"
+ metron_jdbc_password: "root"
+ metron_jdbc_platform: "h2"
+ - kibana-env:
+ kibana_pid_dir: /var/run/kibana
+ kibana_es_url: http://{{ groups.web[0] }}:9200
+ kibana_log_dir: /var/log/kibana
+ kibana_server_port: 5000
+ kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
+ kibana_server_host: 0.0.0.0
+
+blueprint:
+ stack_name: HDP
+ stack_version: "{{ hdp_stack }}"
+ required_configurations: "{{ required_configurations }}"
+ groups:
+ - name : master_1
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ master_1_components }}"
+ hosts: "{{ master_1_host }}"
+ - name : master_2
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ master_2_components }}"
+ hosts: "{{ master_2_host }}"
+ - name : metron
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ metron_components }}"
+ hosts: "{{ metron_host }}"
+ - name : web
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ web_components }}"
+ hosts: "{{ web_host }}"
+ - name: slaves
+ cardinality: 1+
+ configuration: [] # configuration not yet implemented
+ components: "{{ slave_components }}"
+ hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml
new file mode 100644
index 0000000..61197e3
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_gather_facts/meta/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - epel
+ - python-pip
+ - httplib2
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
new file mode 100644
index 0000000..2b37eec
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
@@ -0,0 +1,234 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# cluster_name
+#
+- name: "Ask Ambari: cluster_name"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: cluster_name_response
+ when: cluster_name is undefined
+
+- set_fact:
+ cluster_name: "{{ (cluster_name_response.content | from_json)['items'][0].Clusters.cluster_name }}"
+ when: cluster_name is undefined
+
+#
+# namenode_host
+#
+- name: "Ask Ambari: namenode_host"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: namenode_hosts_response
+ when: namenode_host is undefined
+
+- set_fact:
+ namenode_host: "{{ (namenode_hosts_response.content | from_json).host_components[0].HostRoles.host_name }}"
+ when: namenode_host is undefined
+
+#
+# core_site_tag
+#
+- name: "Ask Ambari: core_site_tag"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: core_site_tag_response
+ when: core_site_tag is undefined
+
+- set_fact:
+ core_site_tag: "{{ (core_site_tag_response.content | from_json).HostRoles.actual_configs['core-site'].default }}"
+ when: core_site_tag is undefined
+
+#
+# hdfs_url
+#
+- name: "Ask Ambari: hdfs_url"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: core_site_response
+ when: hdfs_url is undefined
+
+- set_fact:
+ hdfs_url: "{{ (core_site_response.content | from_json)['items'][0].properties['fs.defaultFS'] }}"
+ when: hdfs_url is undefined
+
+#
+# kafka_broker_hosts
+#
+- name: "Ask Ambari: kafka_broker_hosts"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: kafka_broker_hosts_response
+ when: kafka_broker_hosts is undefined
+
+- set_fact:
+ kafka_broker_hosts: "{{ (kafka_broker_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
+ when: kafka_broker_hosts is undefined
+
+#
+# kafka_broker_tag
+#
+- name: "Ask Ambari: kafka_broker_tag"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: kafka_broker_tag_response
+ when: kafka_broker_tag is undefined
+
+- set_fact:
+ kafka_broker_tag: "{{ (kafka_broker_tag_response.content | from_json).HostRoles.actual_configs['kafka-broker'].default }}"
+ when: kafka_broker_tag is undefined
+
+#
+# kafka_broker_port
+#
+- name: "Ask Ambari: kafka_broker_port"
+ shell: >
+ curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H "X-Requested-By: ambari" "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}" | python -c 'import sys, json; print json.load(sys.stdin)["items"][0]["properties"]["listeners"]'
+ args:
+ warn: false
+ register: kafka_broker_port_response
+ when: (kafka_broker_url is undefined) or (kafka_broker_port is undefined)
+
+- set_fact:
+ kafka_broker_port: "{{ kafka_broker_port_response.stdout_lines[0] | replace('PLAINTEXT://localhost:', '')}}"
+ when: kafka_broker_port is undefined
+
+- set_fact:
+ kafka_broker_url: "{% for host in kafka_broker_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ kafka_broker_port }}{% endfor %}"
+ when: kafka_broker_url is undefined
+
+#
+# zookeeper_hosts
+#
+- name: "Ask Ambari: zookeeper_hosts"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: zookeeper_hosts_response
+ when: zookeeper_hosts is undefined
+
+- set_fact:
+ zookeeper_hosts: "{{ (zookeeper_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
+ when: zookeeper_hosts is undefined
+
+#
+# zookeeper_tag
+#
+- name: "Ask Ambari: zookeeper_tag"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: zookeeper_tag_response
+ when: zookeeper_tag is undefined
+
+- set_fact:
+ zookeeper_tag: "{{ (zookeeper_tag_response.content | from_json).HostRoles.actual_configs['zoo.cfg'].default }}"
+ when: zookeeper_tag is undefined
+
+#
+# zookeeper_url, zookeeper_port
+#
+- name: "Ask Ambari: zookeeper_url, zookeeper_port"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: zookeeper_properties_response
+ when: zookeeper_url is undefined or zookeeper_port is undefined
+
+- set_fact:
+ zookeeper_port: "{{ (zookeeper_properties_response.content | from_json)['items'][0].properties['clientPort'] }}"
+ when: zookeeper_port is undefined
+
+- set_fact:
+ zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}"
+ when: zookeeper_url is undefined
+
+- name: "Ask Ambari: metron_hosts"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: metron_hosts_response
+ when: metron_hosts is undefined
+
+- set_fact:
+ metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
+ when: metron_hosts is undefined
+
+- name: "Ask Ambari: kibana_hosts"
+ uri:
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER"
+ user: "{{ ambari_user }}"
+ password: "{{ ambari_password }}"
+ force_basic_auth: yes
+ return_content: yes
+ register: kibana_hosts_response
+ when: kibana_hosts is undefined
+
+- set_fact:
+ kibana_hosts: "{{ (kibana_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
+ when: kibana_hosts is undefined
+
+
+#
+# debug output
+#
+- name: debug
+ debug:
+ msg: "zookeeper_port = {{ zookeeper_port }},
+ zookeeper_hosts = {{ zookeeper_hosts }},
+ zookeeper_url = {{ zookeeper_url }},
+ kafka_broker_port = {{ kafka_broker_port }},
+ kafka_broker_hosts = {{ kafka_broker_hosts }},
+ kafka_broker_url = {{ kafka_broker_url }},
+ metron_hosts = {{ metron_hosts }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/defaults/main.yml b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
new file mode 100644
index 0000000..0719b7f
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_master/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ambari_server_mem: 2048
+ambari_mpack_version: 0.4.3.0
+metron_mpack_name: metron_mpack-{{ ambari_mpack_version }}.tar.gz
+metron_mpack_path: "{{ playbook_dir }}/../../packaging/ambari/metron-mpack/target/{{ metron_mpack_name }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml b/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml
new file mode 100644
index 0000000..ba55f30
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/ambari.yml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install Ambari Server on CentOS
+ yum: name=ambari-server
+ when: ansible_distribution == "CentOS"
+
+- name: Install Ambari Server on Ubuntu
+ apt: name=ambari-server force=yes
+ when: ansible_distribution == "Ubuntu"
+
+- name: Set Ambari Server Max Memory
+ replace:
+ dest: /var/lib/ambari-server/ambari-env.sh
+ regexp: "\ -Xmx2048m\ "
+ replace: " -Xmx{{ ambari_server_mem }}m "
+ backup: no
+
+- name: Setup Ambari Server
+ shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured
+ register: ambari_server_setup
+ failed_when: "(ambari_server_setup.stderr is defined and ambari_server_setup.stderr != '') or 'FATAL' in ambari_server_setup.stdout"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/main.yml b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
new file mode 100644
index 0000000..77a2e74
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: ambari.yml
+- include: mpack.yml
+
+- name: start ambari server
+ service:
+ name: ambari-server
+ state: restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml b/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml
new file mode 100644
index 0000000..16289bb
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_master/tasks/mpack.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Copy MPack to Ambari Host
+ copy:
+ src: "{{ metron_mpack_path }}"
+ dest: /tmp
+
+- name: Install MPack on Ambari Host
+ shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-{{ ambari_mpack_version }}.tar.gz
+ args:
+ creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml b/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml
new file mode 100644
index 0000000..f3cb4c5
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_slave/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ambari_installation_user: "root"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh b/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh
new file mode 100644
index 0000000..cc8c1cd
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_slave/files/hostname.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo {{ inventory_hostname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_slave/meta/main.yml b/metron-deployment/ansible/roles/ambari_slave/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_slave/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
new file mode 100644
index 0000000..62d0027
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install ambari-agent on CentOS
+ yum: name=ambari-agent
+ when: ansible_distribution == "CentOS"
+
+- name: Install ambari-agent on Ubuntu
+ apt: name=ambari-agent force=yes
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
new file mode 100644
index 0000000..b78a2a7
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: install-agent.yml
+
+- name: Create ambari-agent hostname script
+ template:
+ src: "../roles/ambari_slave/files/hostname.sh"
+ dest: "/var/lib/ambari-agent/hostname.sh"
+ mode: 0744
+ owner: "{{ ambari_installation_user }}"
+ group: "{{ ambari_installation_user }}"
+
+- name: Configure ambari-server hostname in ambari-agent configuration
+ lineinfile:
+ dest: /etc/ambari-agent/conf/ambari-agent.ini
+ regexp: "{{ item.regexp }}"
+ line: "{{ item.line }}"
+ insertafter: "{{ item.insertafter }}"
+ backup: yes
+ with_items:
+ - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
+ - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
+
+- name: Ensure ambari-agent is running
+ service:
+ name: ambari-agent
+ state: restarted
+ enabled: yes
+
+- name : Wait for agent to register
+ command : sleep 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/meta/main.yml b/metron-deployment/ansible/roles/bro/meta/main.yml
new file mode 100644
index 0000000..df226e7
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/meta/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - libselinux-python
+ - build-tools
+ - kafka-client
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/bro.yml b/metron-deployment/ansible/roles/bro/tasks/bro.yml
new file mode 100644
index 0000000..222ef0e
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/tasks/bro.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download bro
+ get_url:
+ url: "https://www.bro.org/downloads/bro-{{ bro_version }}.tar.gz"
+ dest: "/tmp/bro-{{ bro_version }}.tar.gz"
+
+- name: Extract bro tarball
+ unarchive:
+ src: "/tmp/bro-{{ bro_version }}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/bro-{{ bro_version }}"
+
+- name: Compile and Install bro
+ shell: "{{ item }}"
+ environment:
+ CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
+ CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+ args:
+ chdir: "/tmp/bro-{{ bro_version }}"
+ creates: "{{ bro_home }}/bin/bro"
+ with_items:
+ - "./configure --prefix={{ bro_home }}"
+ - make
+ - make install
+
+- name: Configure bro
+ lineinfile:
+ dest: "{{ bro_home }}/etc/node.cfg"
+ regexp: '^interface=.*$'
+ line: 'interface={{ sniff_interface }}'
+
+- name: Install bro configuration
+ shell: "{{ bro_home }}/bin/broctl install"
+
+- name: Bro Cronjob
+ cron:
+ name: Bro Cron
+ minute: "{{ bro_crontab_minutes }}"
+ job: "{{ item }}"
+ with_items:
+ - "{{ bro_crontab_job }}"
+ - "{{ bro_clean_job }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/dependencies.yml b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
new file mode 100644
index 0000000..3cd3bae
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/tasks/dependencies.yml
@@ -0,0 +1,48 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install yum repositories
+ yum: name={{ item }} update_cache=yes
+ with_items:
+ - centos-release-scl
+
+- name: Install prerequisites
+ yum: name={{ item }}
+ with_items:
+ - cmake
+ - make
+ - gcc
+ - gcc-c++
+ - flex
+ - bison
+ - libpcap
+ - libpcap-devel
+ - openssl-devel
+ - python-devel
+ - swig
+ - zlib-devel
+ - perl
+ - crontabs
+ - net-tools
+ - devtoolset-4-gcc
+ - devtoolset-4-gcc-c++
+ - python27
+ - rh-git29
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml b/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml
new file mode 100644
index 0000000..652d319
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/tasks/librdkafka.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download librdkafka
+ get_url:
+ url: "{{ librdkafka_url }}"
+ dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+
+- name: Extract librdkafka tarball
+ unarchive:
+ src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/librdkafka-{{ librdkafka_version }}"
+
+- name: Compile and install librdkafka
+ shell: "{{ item }}"
+ args:
+ chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
+ creates: "{{ librdkafka_home }}/lib/librdkafka.so"
+ with_items:
+ - rm -rf build/
+ - "./configure --prefix={{ librdkafka_home }}"
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/main.yml b/metron-deployment/ansible/roles/bro/tasks/main.yml
new file mode 100644
index 0000000..440e4b5
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: librdkafka.yml
+- include: bro.yml
+- include: metron-bro-plugin-kafka.yml
+- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
new file mode 100644
index 0000000..7043387
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/tasks/metron-bro-plugin-kafka.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install bro-pkg
+ environment:
+ LD_LIBRARY_PATH: "{{ python27_lib }}"
+ command: "{{ python27_bin }}/pip install bro-pkg"
+
+- name: Configure bro-pkg
+ environment:
+ PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
+ LD_LIBRARY_PATH: "{{ python27_lib }}"
+ command: "{{ python27_bin }}/bro-pkg autoconfig"
+
+- name: Install the metron-bro-plugin-kafka package
+ environment:
+ PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
+ LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
+ CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
+ CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
+ command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}"
+
+- name: Configure bro-kafka plugin
+ lineinfile:
+ dest: "{{ bro_home }}/share/bro/site/local.bro"
+ line: "{{ item }}"
+ with_items:
+ - "@load packages"
+ - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);"
+ - "redef Kafka::topic_name = \"{{ bro_topic }}\";"
+ - "redef Kafka::tag_json = T;"
+ - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");"
+
+- name: Deploy bro configuration changes
+ shell: "{{ bro_bin }}/broctl deploy"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/tasks/nic.yml b/metron-deployment/ansible/roles/bro/tasks/nic.yml
new file mode 100644
index 0000000..6053618
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/tasks/nic.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Turn on promiscuous mode for {{ sniff_interface }}
+ shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/bro/vars/main.yml b/metron-deployment/ansible/roles/bro/vars/main.yml
new file mode 100644
index 0000000..11d6c31
--- /dev/null
+++ b/metron-deployment/ansible/roles/bro/vars/main.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+bro_home: /usr/local/bro
+bro_bin: "{{ bro_home }}/bin"
+bro_version: 2.5.2
+bro_daemon_log: /var/log/bro.log
+bro_topic: bro
+
+# Bro cronjob
+bro_crontab_minutes: 0-59/5
+bro_crontab_job: "{{ bro_home }}/bin/broctl cron"
+bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*"
+
+# Bro kafka plugin
+metron_bro_plugin_kafka_version: "0.1"
+python27_bin: /opt/rh/python27/root/usr/bin
+python27_lib: /opt/rh/python27/root/usr/lib64
+git29_bin: /opt/rh/rh-git29/root/usr/bin
+httpd24_lib: /opt/rh/httpd24/root/usr/lib64
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/build-tools/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/build-tools/meta/main.yml b/metron-deployment/ansible/roles/build-tools/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/metron-deployment/ansible/roles/build-tools/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/build-tools/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/build-tools/tasks/main.yml b/metron-deployment/ansible/roles/build-tools/tasks/main.yml
new file mode 100644
index 0000000..c47ef43
--- /dev/null
+++ b/metron-deployment/ansible/roles/build-tools/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install Build Tools
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - libdnet-devel
+ - rpm-build
+ - libpcap
+ - libpcap-devel
+ - pcre
+ - pcre-devel
+ - zlib
+ - zlib-devel
+ - glib2-devel
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/deployment-report/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/deployment-report/defaults/main.yml b/metron-deployment/ansible/roles/deployment-report/defaults/main.yml
new file mode 100644
index 0000000..7677f48
--- /dev/null
+++ b/metron-deployment/ansible/roles/deployment-report/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+timeout_secs: 120
+ambari_host: "{{ groups.ambari_master[0] }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/deployment-report/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/deployment-report/meta/main.yml b/metron-deployment/ansible/roles/deployment-report/meta/main.yml
new file mode 100644
index 0000000..c3d807b
--- /dev/null
+++ b/metron-deployment/ansible/roles/deployment-report/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/deployment-report/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/deployment-report/tasks/main.yml b/metron-deployment/ansible/roles/deployment-report/tasks/main.yml
new file mode 100644
index 0000000..38a5289
--- /dev/null
+++ b/metron-deployment/ansible/roles/deployment-report/tasks/main.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the 'License'); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an 'AS IS' BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Sanity check Ambari web
+ local_action: wait_for host="{{ ambari_host }}" port="{{ ambari_port }}" timeout="{{ timeout_secs }}"
+
+- name: Known hosts groups
+ debug: var=groups
+
+- set_fact:
+ success:
+ - "Apache Metron deployed successfully"
+ - " Ambari @ http://{{ ambari_host }}:{{ ambari_port }}"
+ - " Zookeeper @ {{ zookeeper_url }}"
+ - " Kafka @ {{ kafka_broker_url }}"
+ - For additional information, see https://metron.apache.org/'
+
+- debug: var=success
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/enable-swap/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/enable-swap/defaults/main.yml b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml
new file mode 100644
index 0000000..7190606
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+swapfile: /swapfile
+swapspace: 4G
[09/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/README.md b/metron-deployment/vagrant/README.md
deleted file mode 100644
index 6a86b6a..0000000
--- a/metron-deployment/vagrant/README.md
+++ /dev/null
@@ -1,21 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Vagrant Deployment
-
-- Fast CAPA Test Platform
-- Full Dev Platform
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/.gitignore b/metron-deployment/vagrant/fastcapa-test-platform/.gitignore
deleted file mode 100644
index a8b42eb..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.retry
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/README.md b/metron-deployment/vagrant/fastcapa-test-platform/README.md
deleted file mode 100644
index 213d115..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/README.md
+++ /dev/null
@@ -1,138 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-Fastcapa Test Environment
-=========================
-
-Provides a test environment for the development and testing of Fastcapa. The environment is automatically validated after it is created to ensure that Fastcapa is behaving correctly.
-
-Two virtualized nodes are launched with Vagrant that can communicate with one another over a private network.
-- The `source` node uses Metron's `pcap_replay` functionality to transmit raw network packet data over a private network.
-- The `sink` node is running `fastcapa` and is capturing these network packets.
-- Fastcapa then transforms and bundles the packets into a message.
-- The message is sent to a Kafka broker running on the `source` node.
-
-Getting Started
----------------
-
-The Fastcapa test environment can be executed on different operating systems. There is a sub-directory for each operating system that Fastcapa can be tested on.
-
-To run, simply execute `vagrant up` within the appropriate directory. For example, to run the tests on CentOS 7.1 then execute the following commands.
-```
-cd centos-7.1
-vagrant up
-```
-
-Automated tests are executed after provisioning completes to ensure that Fastcapa and the rest of the environment is functioning properly. If you see something like the following, then the tests have passed.
-```
-$ vagrant up
-==> source: Running provisioner: ansible...
- source: Running ansible-playbook...
-...
-TASK [debug] *******************************************************************
-ok: [source] => {
- "msg": "Successfully received packets sent from pcap-replay!"
-}
-...
-TASK [debug] *******************************************************************
-ok: [source] => {
- "msg": "Successfully received a Kafka message from fastcapa!"
-}
-```
-
-If the deployment process fails mid-course, running `vagrant provision` will continue the process from where it left off. This can sometimes occur when the VM reboots as part of the deployment process. The error might look like the following.
-```
-TASK [fastcapa : Restart for modified kernel params] ***************************
-fatal: [sink]: UNREACHABLE! => {"changed": false, "msg": "Failed to connect to the host via ssh: Shared connection to 127.0.0.1 closed.\r\n", "unreachable": true}
- to retry, use: --limit @/Users/nallen/Development/metron/metron-deployment/vagrant/fastcapa-test-platform/playbook.retry
-
-PLAY RECAP *********************************************************************
-sink : ok=11 changed=9 unreachable=1 failed=0
-source : ok=29 changed=25 unreachable=0 failed=0
-
-Ansible failed to complete successfully. Any error output should be
-visible above. Please fix these errors and try again.
-```
-
-Going Deeper
-------------
-
-This section will outline in more detail the environment and how to interact with it.
-
-### `source`
-
-To validate that the `source` node is functioning properly, run the following commands.
-
-First, ensure that the `pcap-replay` service is running.
-
-```
-vagrant ssh source
-sudo service pcap-replay status
-```
-
-Use `tcpdump` to ensure that the raw packet data is being sent over the private network. Enter 'CTRL-C' to kill the `tcpdump` process once you are able to see that packets are being sent.
-
-```
-sudo yum -y install tcpdump
-sudo tcpdump -i enp0s8
-```
-
-### `sink`
-
-Next validate that the `sink` is functioning properly. Run the following commands starting from the host operating system.
-
-First, ensure that the `fastcapa` service is running.
-
-```
-vagrant ssh sink
-service fastcapa status
-```
-
-Ensure that the raw network packet data is being received by Kafka.
-
-```
-/usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic pcap
-```
-
-Enter 'CTRL-C' to kill the `kafka-console-consumer` process once you are able to see that packets are being sent. These packets will appear to be gibberish in the console. This is the raw binary network packet data after all.
-
-FAQ
----
-
-### Error Message: Timed out while waiting for the machine to boot
-
-```
-Timed out while waiting for the machine to boot. This means that
-Vagrant was unable to communicate with the guest machine within
-the configured ("config.vm.boot_timeout" value) time period.
-If you look above, you should be able to see the error(s) that
-Vagrant had when attempting to connect to the machine. These errors
-are usually good hints as to what may be wrong.
-If you're using a custom box, make sure that networking is properly
-working and you're able to connect to the machine. It is a common
-problem that networking isn't setup properly in these boxes.
-Verify that authentication configurations are also setup properly,
-as well.
-If the box appears to be booting properly, you may want to increase
-the timeout ("config.vm.boot_timeout") value.
-➜ centos-7.4 git:(master) ✗ vagrant status
-Current machine states:
-source running (virtualbox)
-sink not created (virtualbox)
-```
-
-If you are unable to launch any of the Fastcapa test environments, which results in a message like the one above, then you may need to upgrade your version of Virtualbox. Success has been reported with versions of VirtualBox 5.1.22+.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/Vagrantfile b/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/Vagrantfile
deleted file mode 100644
index 179ca34..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/Vagrantfile
+++ /dev/null
@@ -1,72 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Vagrant.configure("2") do |config|
-
- # enable hostmanager
- config.hostmanager.enabled = true
- config.hostmanager.manage_host = true
-
- #
- # source
- #
- config.vm.define "source" do |node|
-
- # host settings
- node.vm.hostname = "source"
- node.vm.box = "bento/centos-7.1"
- node.ssh.insert_key = "true"
- node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
-
- # provider
- node.vm.provider "virtualbox" do |vb|
- vb.memory = 1024
- vb.cpus = 1
- end
- end
-
- #
- # sink
- #
- config.vm.define "sink" do |node|
-
- # host settings
- node.vm.hostname = "sink"
- node.vm.box = "bento/centos-7.1"
- node.ssh.insert_key = "true"
- node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
-
- # provider
- node.vm.provider "virtualbox" do |vb|
- vb.memory = 4096
- vb.cpus = 3
-
- # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
- vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
- vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
- end
-
- # provision host
- node.vm.provision :ansible do |ansible|
- ansible.limit = "all"
- ansible.playbook = "../playbook.yml"
- ansible.extra_vars = "vars/main.yml"
- end
- end
-
-
-end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/ansible.cfg b/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/ansible.cfg
deleted file mode 100644
index 0a83e32..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/ansible.cfg
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[defaults]
-host_key_checking = false
-library = ../../../extra_modules
-roles_path = ../../../roles
-pipelining = True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/vars/main.yml b/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/vars/main.yml
deleted file mode 100644
index 563fc2a..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.1/vars/main.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-# 0.9.4+ required for fastcapa
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
-
-dpdk_device: ["00:08.0"]
-dpdk_target: "x86_64-native-linuxapp-gcc"
-num_huge_pages: 512
-kafka_broker_url: source:9092
-zookeeper_url: source:2181
-pcap_replay_interface: enp0s3
-kafka_broker_home: /usr/hdp/current/kafka-broker/
-
-# fastcapa settings
-fastcapa_portmask: 0x01
-fastcapa_kafka_config: /etc/fastcapa.conf
-fastcapa_topic: pcap
-fastcapa_burst_size: 32
-fastcapa_nb_rx_desc: 1024
-fastcapa_nb_rx_queue: 1
-fastcapa_tx_ring_size: 2048
-
-# dummy variables for pycapa's dependence on ambari_gather_facts
-cluster_name: dummy
-namenode_host: dummy
-core_site_tag: dummy
-hdfs_url: dummy
-kafka_broker_hosts: dummy
-kafka_broker_tag: dummy
-kafka_broker_port: dummy
-zookeeper_hosts: dummy
-zookeeper_tag: dummy
-zookeeper_port: dummy
-metron_hosts: dummy
-kibana_hosts: dummy
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/Vagrantfile b/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/Vagrantfile
deleted file mode 100644
index c62a0f1..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/Vagrantfile
+++ /dev/null
@@ -1,72 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Vagrant.configure("2") do |config|
-
- # enable hostmanager
- config.hostmanager.enabled = true
- config.hostmanager.manage_host = true
-
- #
- # source
- #
- config.vm.define "source" do |node|
-
- # host settings
- node.vm.hostname = "source"
- node.vm.box = "bento/centos-7.4"
- node.ssh.insert_key = "true"
- node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
-
- # provider
- node.vm.provider "virtualbox" do |vb|
- vb.memory = 1024
- vb.cpus = 1
- end
- end
-
- #
- # sink
- #
- config.vm.define "sink" do |node|
-
- # host settings
- node.vm.hostname = "sink"
- node.vm.box = "bento/centos-7.4"
- node.ssh.insert_key = "true"
- node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
-
- # provider
- node.vm.provider "virtualbox" do |vb|
- vb.memory = 4096
- vb.cpus = 3
-
- # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
- vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
- vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
- end
-
- # provision host
- node.vm.provision :ansible do |ansible|
- ansible.limit = "all"
- ansible.playbook = "../playbook.yml"
- ansible.extra_vars = "vars/main.yml"
- end
- end
-
-
-end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/ansible.cfg b/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/ansible.cfg
deleted file mode 100644
index 0a83e32..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/ansible.cfg
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[defaults]
-host_key_checking = false
-library = ../../../extra_modules
-roles_path = ../../../roles
-pipelining = True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/vars/main.yml b/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/vars/main.yml
deleted file mode 100644
index 563fc2a..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/centos-7.4/vars/main.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-# 0.9.4+ required for fastcapa
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
-
-dpdk_device: ["00:08.0"]
-dpdk_target: "x86_64-native-linuxapp-gcc"
-num_huge_pages: 512
-kafka_broker_url: source:9092
-zookeeper_url: source:2181
-pcap_replay_interface: enp0s3
-kafka_broker_home: /usr/hdp/current/kafka-broker/
-
-# fastcapa settings
-fastcapa_portmask: 0x01
-fastcapa_kafka_config: /etc/fastcapa.conf
-fastcapa_topic: pcap
-fastcapa_burst_size: 32
-fastcapa_nb_rx_desc: 1024
-fastcapa_nb_rx_queue: 1
-fastcapa_tx_ring_size: 2048
-
-# dummy variables for pycapa's dependence on ambari_gather_facts
-cluster_name: dummy
-namenode_host: dummy
-core_site_tag: dummy
-hdfs_url: dummy
-kafka_broker_hosts: dummy
-kafka_broker_tag: dummy
-kafka_broker_port: dummy
-zookeeper_hosts: dummy
-zookeeper_tag: dummy
-zookeeper_port: dummy
-metron_hosts: dummy
-kibana_hosts: dummy
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml b/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
deleted file mode 100644
index e038691..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/playbook.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# the 'source' produces network traffic
-#
-- hosts: source
- become: yes
- roles:
- - role: kafka-broker
- - role: pcap_replay
- - { role: pycapa, install_pycapa_service: False }
- - { role: sensor-test-mode, pcap_replay: True, install_yaf: False, install_snort: False, install_pcap_replay: True }
- tasks:
- - service: name=pcap-replay state=started
-
-#
-# the 'sink' consumes network traffic
-#
-- hosts: sink
- become: yes
- roles:
- - role: librdkafka
- - role: fastcapa
- tasks:
- - service: name=fastcapa state=started
-
-#
-# validate the environment - needs to run on `source` node
-#
-- hosts: source
- become: yes
- tasks:
- - include: tasks/validate-packets-sent.yml
- - include: tasks/validate-messages-received.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-messages-received.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-messages-received.yml b/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-messages-received.yml
deleted file mode 100644
index 5109a0e..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-messages-received.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- set_fact:
- kafka_consumer_cmd: "{{ kafka_broker_home}}/bin/kafka-console-consumer.sh --zookeeper {{ zookeeper_url }} --topic {{ fastcapa_topic }} --max-messages 10 --from-beginning"
-
-- name: Attempting to receive message(s) sent by fastcapa
- shell: "{{ kafka_consumer_cmd }}"
- async: 30
- register: kafka_consumer
-
-- name: Waiting to receive message(s) sent by fastcapa
- async_status: jid={{ kafka_consumer.ansible_job_id }}
- register: job
- until: job.finished
- retries: 30
-
-- fail:
- msg: "No messages received from fastcapa within timeout: {{ kafka_consumer_cmd }}"
- when: job.finished != 1
-
-- debug: var=job
-
-- debug: msg="Successfully received a Kafka message from fastcapa!"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-packets-sent.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-packets-sent.yml b/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-packets-sent.yml
deleted file mode 100644
index 25347dd..0000000
--- a/metron-deployment/vagrant/fastcapa-test-platform/tasks/validate-packets-sent.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install tcpdump
- yum: name=tcpdump
-
-- set_fact:
- sniff_cmd: "tcpdump -i {{ pcap_replay_interface }} -c 10"
-
-- name: "Attempting to sniff packet(s)"
- shell: "{{ sniff_cmd }}"
- async: 30
- register: sniffer
-
-- name: Waiting to receive packet(s)
- async_status: jid={{ sniffer.ansible_job_id }}
- register: job
- until: job.finished
- retries: 30
-
-- fail:
- msg: "No packets received from pcap-replay within timeout: {{ sniff_cmd }}"
- when: job.finished != 1
-
-- debug: var=job
-
-- debug: msg="Successfully received packets sent from pcap-replay!"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/full-dev-platform/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/.gitignore b/metron-deployment/vagrant/full-dev-platform/.gitignore
deleted file mode 100644
index 8000dd9..0000000
--- a/metron-deployment/vagrant/full-dev-platform/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-.vagrant
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/full-dev-platform/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/README.md b/metron-deployment/vagrant/full-dev-platform/README.md
deleted file mode 100644
index 4a58915..0000000
--- a/metron-deployment/vagrant/full-dev-platform/README.md
+++ /dev/null
@@ -1,123 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-Full Development Platform
-=========================
-
-This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running on Virtualbox.
-
-Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services. To that end the vagrant file configuration defaults to disabling solr and yaf.
-
-Getting Started
----------------
-
-### Prerequisites
-
-The computer used to deploy Apache Metron will need to have the following components installed.
-
- - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
- - [Docker](https://www.docker.com/community-edition)
- - [Vagrant](https://www.vagrantup.com) 1.8+
- - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
- - [Virtualbox](https://virtualbox.org) 5.0+
- - Python 2.7
- - Maven 3.3.9
- - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11)
-
-Running the following script can help validate whether you have all the prerequisites installed and running correctly.
-
- ```
- metron-deployment/scripts/platform-info.sh
- ```
-
-#### How do I install these on MacOS?
-
-Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project.
-
-1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/).
-
-1. Run the following command in a terminal to install all of the required tools.
-
- ```
- brew cask install vagrant virtualbox docker
- brew cask install caskroom/versions/java8
- brew install maven@3.3 git
- pip install ansible==2.2.2.0
- vagrant plugin install vagrant-hostmanager
- open /Applications/Docker.app
- ```
-
-### Deploy Metron
-
-1. Ensure that the Docker service is running.
-
-1. Deploy Metron
-
- ```
- cd metron-deployment/vagrant/full-dev-platform
- vagrant up
- ```
-
- Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.
-
- ```
- vagrant provision
- ```
-
-### Explore Metron
-
-Navigate to the following resources to explore your newly minted Apache Metron environment.
-
- - [Ambari](http://node1:8080)
- - [Metron](http://node1:5000)
- - [Services](http://node1:2812)
-
-Connecting to the host through SSH is as simple as running the following command.
-
-```
-vagrant ssh
-```
-
-### Working with Metron
-
-In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `web` role on the Vagrant image. This will install components (if necessary) and start the UI.
-
-```
-./run_ansible_role.sh web
-```
-or
-
-```
-vagrant --ansible-tags="web" provision
-```
-
-#### Using Tags
-
-A collection of tags is specified as a comma separated list.
-
-```
-./run_ansible_role.sh "sensors,enrichment"
-
-```
-
-Tags are listed in the playbooks, some frequently used tags:
-+ `hdp-install` - Install HDP
-+ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
-+ `sensors` - Deploy and Start Sensors.
-+ `enrichment` - Deploy and Start Enrichment Topology.
-
-Note also that there is a convenience script `./run_enrichment_role.sh` which executes Vagrant with the `enrichment` tag.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/full-dev-platform/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/Vagrantfile b/metron-deployment/vagrant/full-dev-platform/Vagrantfile
deleted file mode 100644
index 15df777..0000000
--- a/metron-deployment/vagrant/full-dev-platform/Vagrantfile
+++ /dev/null
@@ -1,92 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-require 'getoptlong'
-
-ansibleTags=''
-ansibleSkipTags='sensors'
-
-begin
- opts = GetoptLong.new(
- [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ],
- [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ]
- )
-
- opts.quiet = TRUE
-
- opts.each do |opt, arg|
- case opt
- when '--ansible-tags'
- ansibleTags=arg
- when '--ansible-skip-tags'
- ansibleSkipTags=arg
- end
- end
-rescue Exception => ignored
-#Ignore to allow other opts to be passed to Vagrant
-end
-
-puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != ''
-puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != ''
-
-hosts = [{
- hostname: "node1",
- ip: "192.168.66.121",
- memory: "8192",
- cpus: 4,
- promisc: 2 # enables promisc on the 'Nth' network interface
-}]
-
-Vagrant.configure(2) do |config|
-
- # all hosts built on centos 6
- config.vm.box = "metron/centos_base"
- config.ssh.insert_key = true
-
- # enable the hostmanager plugin
- config.hostmanager.enabled = true
- config.hostmanager.manage_host = true
-
- # host definition
- hosts.each_with_index do |host, index|
- config.vm.define host[:hostname] do |node|
-
- # host settings
- node.vm.hostname = host[:hostname]
- node.vm.network "private_network", ip: host[:ip]
-
- # vm settings
- node.vm.provider "virtualbox" do |vb|
- vb.memory = host[:memory]
- vb.cpus = host[:cpus]
-
- # enable promisc mode on the network interface
- if host.has_key?(:promisc)
- vb.customize ["modifyvm", :id, "--nicpromisc#{host[:promisc]}", "allow-all"]
- end
- end
- end
- end
-
- # provisioning
- config.vm.provision :ansible do |ansible|
- ansible.playbook = "../../playbooks/metron_full_install.yml"
- ansible.sudo = true
- ansible.tags = ansibleTags.split(",") if ansibleTags != ''
- ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
- ansible.inventory_path = "../../inventory/full-dev-platform"
- end
-end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/full-dev-platform/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/ansible.cfg b/metron-deployment/vagrant/full-dev-platform/ansible.cfg
deleted file mode 100644
index 74f3fdf..0000000
--- a/metron-deployment/vagrant/full-dev-platform/ansible.cfg
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-[defaults]
-host_key_checking = false
-library = ../../extra_modules
-roles_path = ../../roles
-pipelining = True
-log_path = ./ansible.log
-
-
-# fix for "ssh throws 'unix domain socket too long' " problem
-[ssh_connection]
-control_path = %(directory)s/%%h-%%p-%%r
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/full-dev-platform/run_ansible_role.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/run_ansible_role.sh b/metron-deployment/vagrant/full-dev-platform/run_ansible_role.sh
deleted file mode 100755
index 0e5d831..0000000
--- a/metron-deployment/vagrant/full-dev-platform/run_ansible_role.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-vagrant --ansible-tags="$@" provision
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/vagrant/full-dev-platform/run_enrichment_role.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/vagrant/full-dev-platform/run_enrichment_role.sh b/metron-deployment/vagrant/full-dev-platform/run_enrichment_role.sh
deleted file mode 100755
index e26bc16..0000000
--- a/metron-deployment/vagrant/full-dev-platform/run_enrichment_role.sh
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-./run_ansible_role.sh enrichment
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-interface/metron-rest/README.md
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md
index 3d7d7a1..d51dc52 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -433,7 +433,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Returns sample message
* 404 - Either Kafka topic is missing or contains no messages
-
+
### `POST /api/v1/kafka/topic/{name}/produce`
* Description: Produces a message to a Kafka topic
* Input:
@@ -442,7 +442,7 @@ Request and Response objects are JSON formatted. The JSON schemas are available
* Returns:
* 200 - Published
* 404 - Kafka topic is missing
-
+
### `POST /api/v1/metaalert/searchByAlert`
* Description: Get all meta alerts that contain an alert.
* Input:
@@ -846,7 +846,7 @@ The metron-rest application will be available at http://localhost:8080/swagger-u
### Full Dev
-Start the [Full Dev](../../metron-deployment/vagrant/full-dev-platform) environment. Build the metron-rest module and start it with the Spring Boot Maven plugin:
+Start the [development environment](../../metron-deployment/development/centos6). Build the metron-rest module and start it with the Spring Boot Maven plugin:
```
mvn clean package
mvn spring-boot:run -Drun.profiles=vagrant,dev
[18/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/extra_modules/ambari_service_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/extra_modules/ambari_service_state.py b/metron-deployment/extra_modules/ambari_service_state.py
deleted file mode 100644
index 6d44e8d..0000000
--- a/metron-deployment/extra_modules/ambari_service_state.py
+++ /dev/null
@@ -1,352 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-DOCUMENTATION = '''
----
-module: ambari_service_state
-version_added: "2.1"
-author: Apache Metron (https://metron.apache.org)
-short_description: Start/Stop/Change Service or Component State
-description:
- - Start/Stop/Change Service or Component State
-options:
- host:
- description:
- The hostname for the ambari web server
- port:
- description:
- The port for the ambari web server
- username:
- description:
- The username for the ambari web server
- password:
- description:
- The name of the cluster in web server
- required: yes
- cluster_name:
- description:
- The name of the cluster in ambari
- required: yes
- service_name:
- description:
- The name of the service to alter
- required: no
- component_name:
- description:
- The name of the component to alter
- required: no
- component_host:
- description:
- The host running the targeted component. Required when component_name is used.
- required: no
- state:
- description:
- The desired service/component state.
- wait_for_complete:
- description:
- Whether to wait for the request to complete before returning. Default is False.
- required: no
- requirements: [ 'requests']
-'''
-
-EXAMPLES = '''
-# must use full relative path to any files in stored in roles/role_name/files/
-- name: Create a new ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: present
- blueprint_var: roles/my_role/files/blueprint.yml
- blueprint_name: hadoop
- wait_for_complete: True
-- name: Start the ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: started
- wait_for_complete: True
-- name: Stop the ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: stopped
- wait_for_complete: True
-- name: Delete the ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: absent
-'''
-
-RETURN = '''
-results:
- description: The content of the requests object returned from the RESTful call
- returned: success
- type: string
-'''
-
-__author__ = 'apachemetron'
-
-import json
-
-try:
- import requests
-except ImportError:
- REQUESTS_FOUND = False
-else:
- REQUESTS_FOUND = True
-
-
-def main():
-
- argument_spec = dict(
- host=dict(type='str', default=None, required=True),
- port=dict(type='int', default=None, required=True),
- username=dict(type='str', default=None, required=True),
- password=dict(type='str', default=None, required=True),
- cluster_name=dict(type='str', default=None, required=True),
- state=dict(type='str', default=None, required=True,
- choices=['started', 'stopped', 'deleted']),
- service_name=dict(type='str', required=False),
- component_name=dict(type='str', default=None, required=False),
- component_host=dict(type='str', default=None, required=False),
- wait_for_complete=dict(default=False, required=False, type='bool'),
- )
-
- required_together = ['component_name', 'component_host']
-
- module = AnsibleModule(
- argument_spec=argument_spec,
- required_together=required_together
- )
-
- if not REQUESTS_FOUND:
- module.fail_json(
- msg='requests library is required for this module')
-
- p = module.params
-
- host = p.get('host')
- port = p.get('port')
- username = p.get('username')
- password = p.get('password')
- cluster_name = p.get('cluster_name')
- state = p.get('state')
- service_name = p.get('service_name')
- component_name = p.get('component_name')
- component_host = p.get('component_host')
- wait_for_complete = p.get('wait_for_complete')
- component_mode = False
- ambari_url = 'http://{0}:{1}'.format(host, port)
-
- if component_name:
- component_mode = True
-
- try:
- if not cluster_exists(ambari_url, username, password, cluster_name):
- module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name))
-
- if state in ['started', 'stopped', 'installed']:
- desired_state = ''
-
- if state == 'started':
- desired_state = 'STARTED'
- elif state in ['stopped','installed']:
- desired_state = 'INSTALLED'
-
- if component_mode:
- if desired_state == 'INSTALLED':
- if(can_add_component(ambari_url, username, password, cluster_name, component_name, component_host)):
- add_component_to_host(ambari_url, username, password, cluster_name, component_name, component_host)
- request = set_component_state(ambari_url, username, password, cluster_name, component_name, component_host, desired_state)
- else:
- request = set_service_state(ambari_url,username,password,cluster_name,service_name, desired_state)
- if wait_for_complete:
- try:
- request_id = json.loads(request.content)['Requests']['id']
- except ValueError:
- module.exit_json(changed=True, results=request.content)
- status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
- if status != 'COMPLETED':
- module.fail_json(msg="Request failed with status {0}".format(status))
- module.exit_json(changed=True, results=request.content)
-
- elif state == 'deleted':
- if component_mode:
- request = delete_component(ambari_url, username, password, cluster_name, component_name, component_host)
- else:
- request = delete_service(ambari_url,username,password,cluster_name,service_name)
- module.exit_json(changed=True, results=request.content)
-
- except requests.ConnectionError, e:
- module.fail_json(msg="Could not connect to Ambari client: " + str(e.message))
- except Exception, e:
- module.fail_json(msg="Ambari client exception occurred: " + str(e.message))
-
-
-def get_clusters(ambari_url, user, password):
- r = get(ambari_url, user, password, '/api/v1/clusters')
- if r.status_code != 200:
- msg = 'Could not get cluster list: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- clusters = json.loads(r.content)
- return clusters['items']
-
-
-def cluster_exists(ambari_url, user, password, cluster_name):
- clusters = get_clusters(ambari_url, user, password)
- return cluster_name in [item['Clusters']['cluster_name'] for item in clusters]
-
-
-def get_request_status(ambari_url, user, password, cluster_name, request_id):
- path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id)
- r = get(ambari_url, user, password, path)
- if r.status_code != 200:
- msg = 'Could not get cluster request status: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- service = json.loads(r.content)
- return service['Requests']['request_status']
-
-
-def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time):
- while True:
- status = get_request_status(ambari_url, user, password, cluster_name, request_id)
- if status == 'COMPLETED':
- return status
- elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']:
- return status
- else:
- time.sleep(sleep_time)
-
-
-def set_service_state(ambari_url, user, password, cluster_name, service_name, desired_state):
- path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name)
- request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(service_name,desired_state)},
- "Body": {"ServiceInfo": {"state": "{0}".format(desired_state)}}}
- payload = json.dumps(request)
- r = put(ambari_url, user, password, path, payload)
- if r.status_code not in [202, 200]:
- msg = 'Could not set service state: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def set_component_state(ambari_url, user, password, cluster_name, component_name, component_host, desired_state):
- path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
- request = {"RequestInfo": {"context": "Setting {0} to {1} via REST".format(component_name,desired_state)},
- "Body": {"HostRoles": {"state": "{0}".format(desired_state)}}}
- payload = json.dumps(request)
- r = put(ambari_url, user, password, path, payload)
- if r.status_code not in [202, 200]:
- msg = 'Could not set component state: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def delete_component(ambari_url, user, password, cluster_name, component_name, component_host):
- enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host)
- path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
- r = delete(ambari_url,user,password,path)
- if r.status_code not in [202, 200]:
- msg = 'Could not set service state: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def enable_maint_mode(ambari_url, user, password, cluster_name, component_name, component_host):
- path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
- request = {"RequestInfo":{"context":"Turn On Maintenance Mode for {0}".format(component_name)},
- "Body":{"HostRoles":{"maintenance_state":"ON"}}}
- payload = json.dumps(request)
- r = put(ambari_url, user, password, path, payload)
- if r.status_code not in [202, 200]:
- msg = 'Could not set maintenance mode: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def delete_service(ambari_url, user, password, cluster_name, service_name):
- path = '/api/v1/clusters/{0}/services/{1}'.format(cluster_name,service_name)
- r = delete(ambari_url,user,password,path)
- if r.status_code not in [202, 200]:
- msg = 'Could not delete service: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def add_component_to_host(ambari_url, user, password, cluster_name, component_name, component_host):
- path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
- r = post(ambari_url, user, password, path,'')
- if r.status_code not in [202,201,200]:
- msg = 'Could not add {0} to host {1}: request code {2}, \
- request message {3}'.format(component_name,component_host,r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def can_add_component(ambari_url, user, password, cluster_name, component_name, component_host):
- path = '/api/v1/clusters/{0}/hosts/{1}/host_components/{2}'.format(cluster_name,component_host,component_name)
- r = get(ambari_url, user, password, path)
- return r.status_code == 404
-
-
-def get(ambari_url, user, password, path):
- r = requests.get(ambari_url + path, auth=(user, password))
- return r
-
-
-def put(ambari_url, user, password, path, data):
- headers = {'X-Requested-By': 'ambari'}
- r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers)
- return r
-
-
-def post(ambari_url, user, password, path, data):
- headers = {'X-Requested-By': 'ambari'}
- r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers)
- return r
-
-
-def delete(ambari_url, user, password, path):
- headers = {'X-Requested-By': 'ambari'}
- r = requests.delete(ambari_url + path, auth=(user, password), headers=headers)
- return r
-
-
-from ansible.module_utils.basic import *
-if __name__ == '__main__':
- main()
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/inventory/full-dev-platform/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/group_vars/all b/metron-deployment/inventory/full-dev-platform/group_vars/all
deleted file mode 100644
index c07278b..0000000
--- a/metron-deployment/inventory/full-dev-platform/group_vars/all
+++ /dev/null
@@ -1,89 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# which services should be started?
-services_to_start:
- - snort
- - snort-logs
- - bro
- - pcap-replay
-
-# ambari
-ambari_host: "{{ groups.ambari_master[0] }}"
-hdp_host_group: "{{ groups.ambari_slave }}"
-ambari_port: 8080
-ambari_user: admin
-ambari_password: admin
-cluster_type: single_node_vm
-ambari_server_mem: 512
-java_home: /usr/jdk64/jdk1.8.0_77
-
-# hbase
-pcap_hbase_table: pcap
-tracker_hbase_table: access_tracker
-threatintel_hbase_table: threatintel
-enrichment_hbase_table: enrichment
-
-# metron
-metron_version: 0.4.3
-metron_directory: /usr/metron/{{ metron_version }}
-bro_version: "2.5.2"
-fixbuf_version: "1.7.1"
-yaf_version: "2.8.0"
-daq_version: "2.0.6-1"
-pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
-pycapa_home: "/opt/pycapa"
-snort_version: "2.9.8.0-1"
-snort_alert_csv_path: "/var/log/snort/alert.csv"
-threat_intel_bulk_load: False
-
-# data directories - only required to override defaults
-zookeeper_data_dir: "/data1/hadoop/zookeeper"
-namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
-namenode_name_dir: "/data1/hadoop/hdfs/namenode"
-datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
-journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
-nodemanager_local_dirs: "/data1/hadoop/yarn/local"
-timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
-timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
-nodemanager_log_dirs: "/data1/hadoop/yarn/log"
-jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
-storm_local_dir: "/data1/hadoop/storm"
-kafka_log_dirs: "/data1/kafka-log"
-elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
-
-# sensors
-sensor_test_mode: True
-install_pycapa: False
-install_bro: True
-install_snort: True
-install_yaf: False
-install_pcap_replay: True
-sniff_interface: eth1
-pcap_replay_interface: "{{ sniff_interface }}"
-pcapservice_port: 8081
-
-# search
-install_elasticsearch: True
-install_solr: False
-solr_collection_name: Metron
-solr_number_shards: 1
-solr_replication_factor: 1
-elasticsearch_transport_port: 9300
-## The elasticsearch_network_interface must be in the form specified for Elasticsearch, with leading and trailing underscores.
-elasticsearch_network_interface: _eth1_
-elasticsearch_web_port: 9200
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/inventory/full-dev-platform/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/inventory/full-dev-platform/hosts b/metron-deployment/inventory/full-dev-platform/hosts
deleted file mode 100644
index 9bd9ea1..0000000
--- a/metron-deployment/inventory/full-dev-platform/hosts
+++ /dev/null
@@ -1,47 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-[ambari_master]
-node1
-
-[ambari_slave]
-node1
-
-[metron]
-node1
-
-[search]
-node1
-
-[sensors]
-node1
-
-[pcap_server]
-node1
-
-[web]
-node1
-
-[zeppelin]
-node1
-
-[monit:children]
-sensors
-pcap_server
-
-[local]
-127.0.0.1
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
index fac4f77..9b1888c 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/METRON/CURRENT/repos/repoinfo.xml
@@ -28,5 +28,11 @@
<reponame>METRON</reponame>
</repo>
</os>
+ <os family="ubuntu14">
+ <repo>
+ <baseurl>file:///localrepo</baseurl>
+ <repoid>METRON-${metron.version}</repoid>
+ <reponame>METRON</reponame>
+ </repo>
+ </os>
</reposinfo>
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
index d6db027..cb069b8 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
@@ -55,10 +55,10 @@
<description>Maximum number of memory map areas for process</description>
</property>
- <!--/etc/sysconfig/elasticsearch-->
+ <!-- Elasticsearch sysconfig -->
<property>
<name>content</name>
- <description>This is the jinja template for elastic sysconfig file (/etc/sysconfig/elasticsearch)</description>
+ <description>This is the jinja template for elastic sysconfig file</description>
<value>
# Directory where the Elasticsearch binary distribution resides
ES_HOME={{elastic_home}}
@@ -97,6 +97,9 @@ ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcu
-XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \
-XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \
-Xms{{heap_size}} -Xmx{{heap_size}}"
+
+# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html
+MAX_LOCKED_MEMORY=unlimited
</value>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
new file mode 100644
index 0000000..311e3c0
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-systemd.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>content</name>
+ <description>The jinja template for the Elasticsearch systemd override file. Applies only to platforms that use systemd.</description>
+ <value>
+[Service]
+LimitMEMLOCK=infinity
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
index 3783d31..0943eec 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
@@ -49,13 +49,29 @@
</components>
<osSpecifics>
<osSpecific>
- <osFamily>any</osFamily>
+ <osFamily>redhat6</osFamily>
<packages>
<package>
<name>elasticsearch-5.6.2</name>
</package>
</packages>
</osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu14</osFamily>
+ <packages>
+ <package>
+ <name>elasticsearch=5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
</osSpecifics>
<commandScript>
<script>scripts/service_check.py</script>
@@ -66,6 +82,7 @@
<config-type>elastic-env</config-type>
<config-type>elastic-site</config-type>
<config-type>elastic-sysconfig</config-type>
+ <config-type>elastic-systemd</config-type>
</configuration-dependencies>
<restartRequiredAfterChange>true</restartRequiredAfterChange>
<quickLinksConfigurations>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py
deleted file mode 100755
index e27e8bf..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic.py
+++ /dev/null
@@ -1,86 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-from resource_management.core.resources import User
-from resource_management.core.logger import Logger
-from resource_management.libraries.functions import format as ambari_format
-
-def elastic():
- import params
-
- Logger.info("Creating user: {0}:{1}".format(params.elastic_user, params.elastic_group))
- User(params.elastic_user, action = "create", groups = params.elastic_group)
-
- params.path_data = params.path_data.replace('"', '')
- data_path = params.path_data.replace(' ', '').split(',')
- data_path[:] = [x.replace('"', '') for x in data_path]
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path + ["{0}/scripts".format(params.conf_dir)]
-
- Logger.info("Creating directories: {0}".format(directories))
- Directory(directories,
- create_parents=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_group
- )
-
- Logger.info("Master env: ""{0}/elastic-env.sh".format(params.conf_dir))
- File("{0}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- group=params.elastic_group,
- content=InlineTemplate(params.elastic_env_sh_template)
- )
-
- configurations = params.config['configurations']['elastic-site']
- Logger.info("Master yml: ""{0}/elasticsearch.yml".format(params.conf_dir))
- File("{0}/elasticsearch.yml".format(params.conf_dir),
- content=Template(
- "elasticsearch.master.yaml.j2",
- configurations=configurations),
- owner=params.elastic_user,
- group=params.elastic_group
- )
-
- Logger.info("Master sysconfig: /etc/sysconfig/elasticsearch")
- File("/etc/sysconfig/elasticsearch",
- owner="root",
- group="root",
- content=InlineTemplate(params.sysconfig_template)
- )
-
- # in some OS this folder may not exist, so create it
- Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir))
- Directory(params.limits_conf_dir,
- create_parents=True,
- owner='root',
- group='root'
- )
-
- Logger.info("Master PAM limits: {0}".format(params.limits_conf_file))
- File(params.limits_conf_file,
- content=Template('elasticsearch_limits.conf.j2'),
- owner="root",
- group="root"
- )
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
new file mode 100644
index 0000000..afbaff2
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from ambari_commons.os_check import OSCheck
+from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.exceptions import ComponentIsNotRunning
+from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import Directory
+from resource_management.core.resources.system import File
+from resource_management.core.source import InlineTemplate
+from resource_management.core.source import Template
+from resource_management.core.resources import User
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions import format as ambari_format
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+
+
+def service_check(cmd, user, label):
+ """
+ Executes a SysV service check command that adheres to LSB-compliant
+ return codes. The return codes are interpreted as defined
+ by the LSB.
+
+ See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
+ for more information.
+
+ :param cmd: The service check command to execute.
+ :param label: The name of the service.
+ """
+ Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
+ rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
+
+ if rc in [1, 2, 3]:
+ # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
+ Logger.info("{0} is not running".format(label))
+ raise ComponentIsNotRunning()
+
+ elif rc == 0:
+ # if return code = 0, then 'program is running or service is OK'
+ Logger.info("{0} is running".format(label))
+
+ else:
+ # else service state is unknown
+ err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
+ Logger.error(err_msg)
+ raise ExecutionFailed(err_msg, rc, out, err)
+
+def is_systemd_running():
+ """
+ Determines if the platform is running Systemd.
+ :return True, if the platform is running Systemd. False, otherwise.
+ """
+ Logger.info("Is the platform running Systemd?")
+ rc, out, err = get_user_call_output("pidof systemd", "root", is_checked_call=False)
+ if rc == 0:
+ Logger.info("Systemd was found")
+ return True
+ else:
+ Logger.info("Systemd was NOT found")
+ return False
+
+def configure_systemd(params):
+ """
+ Configure Systemd for Elasticsearch.
+ """
+ Logger.info("Configuring Systemd for Elasticsearch");
+
+ # ensure the systemd directory for elasticsearch overrides exists
+ Logger.info("Create Systemd directory for Elasticsearch: {0}".format(params.systemd_elasticsearch_dir))
+ Directory(params.systemd_elasticsearch_dir,
+ create_parents=True,
+ owner='root',
+ group='root')
+
+ # when using Elasticsearch packages on systems that use systemd, system
+ # limits must also be specified via systemd.
+ # see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
+ Logger.info("Elasticsearch systemd limits: {0}".format(params.systemd_override_file))
+ File(params.systemd_override_file,
+ content=InlineTemplate(params.systemd_override_template),
+ owner="root",
+ group="root")
+
+ # reload the configuration
+ Execute("systemctl daemon-reload")
+
+def create_user(params):
+ """
+ Creates the user required for Elasticsearch.
+ """
+ Logger.info("Creating user={0} in group={1}".format(params.elastic_user, params.elastic_group))
+ User(params.elastic_user, action = "create", groups = params.elastic_group)
+
+def create_directories(params, directories):
+ """
+ Creates one or more directories.
+ """
+ Logger.info("Creating directories: {0}".format(directories))
+ Directory(directories,
+ create_parents=True,
+ mode=0755,
+ owner=params.elastic_user,
+ group=params.elastic_group
+ )
+
+def create_elastic_env(params):
+ """
+ Creates the Elasticsearch environment file.
+ """
+ Logger.info("Create Elasticsearch environment file.")
+ File("{0}/elastic-env.sh".format(params.conf_dir),
+ owner=params.elastic_user,
+ group=params.elastic_group,
+ content=InlineTemplate(params.elastic_env_sh_template))
+
+def create_elastic_site(params, template_name):
+ """
+ Creates the Elasticsearch site file.
+ """
+ Logger.info("Creating Elasticsearch site file; template={0}".format(template_name))
+
+ elastic_site = params.config['configurations']['elastic-site']
+ path = "{0}/elasticsearch.yml".format(params.conf_dir)
+ template = Template(template_name, configurations=elastic_site)
+ File(path,
+ content=template,
+ owner=params.elastic_user,
+ group=params.elastic_group)
+
+def get_elastic_config_path(default="/etc/default/elasticsearch"):
+ """
+ Defines the path to the Elasticsearch environment file. This path will
+ differ based on the OS family.
+ :param default: The path used if the OS family is not recognized.
+ """
+ path = default
+ if OSCheck.is_redhat_family():
+ path = "/etc/sysconfig/elasticsearch"
+ elif OSCheck.is_ubuntu_family():
+ path = "/etc/default/elasticsearch"
+ else:
+ Logger.error("Unexpected OS family; using default path={0}".format(path))
+
+ return path
+
+def create_elastic_config(params):
+ """
+ Creates the Elasticsearch system config file. Usually lands at either
+ /etc/sysconfig/elasticsearch or /etc/default/elasticsearch.
+ """
+ path = get_elastic_config_path()
+ Logger.info("Creating the Elasticsearch system config; path={0}".format(path))
+ File(path, owner="root", group="root", content=InlineTemplate(params.sysconfig_template))
+
+def create_elastic_pam_limits(params):
+ """
+ Creates the PAM limits for Elasticsearch.
+ """
+ Logger.info("Creating Elasticsearch PAM limits.")
+
+ # in some OS this folder may not exist, so create it
+ Logger.info("Ensure PAM limits directory exists: {0}".format(params.limits_conf_dir))
+ Directory(params.limits_conf_dir,
+ create_parents=True,
+ owner='root',
+ group='root')
+
+ Logger.info("Creating Elasticsearch PAM limits; file={0}".format(params.limits_conf_file))
+ File(params.limits_conf_file,
+ content=Template('elasticsearch_limits.conf.j2'),
+ owner="root",
+ group="root")
+
+def get_data_directories(params):
+ """
+ Returns the directories to use for storing Elasticsearch data.
+ """
+ path = params.path_data
+ path = path.replace('"', '')
+ path = path.replace(' ', '')
+ path = path.split(',')
+ dirs = [p.replace('"', '') for p in path]
+
+ Logger.info("Elasticsearch data directories: dirs={0}".format(dirs))
+ return dirs
+
+def configure_master():
+ """
+ Configures the Elasticsearch master node.
+ """
+ import params
+
+ # define the directories required
+ dirs = [
+ params.log_dir,
+ params.pid_dir,
+ params.conf_dir,
+ "{0}/scripts".format(params.conf_dir)
+ ]
+ dirs += get_data_directories(params)
+
+ # configure the elasticsearch master
+ create_user(params)
+ create_directories(params, dirs)
+ create_elastic_env(params)
+ create_elastic_site(params, "elasticsearch.master.yaml.j2")
+ create_elastic_config(params)
+ create_elastic_pam_limits(params)
+ if is_systemd_running():
+ configure_systemd(params)
+
+def configure_slave():
+ """
+ Configures the Elasticsearch slave node.
+ """
+ import params
+
+ # define the directories required
+ dirs = [
+ params.log_dir,
+ params.pid_dir,
+ params.conf_dir,
+ ]
+ dirs += get_data_directories(params)
+
+ # configure the elasticsearch slave
+ create_user(params)
+ create_directories(params, dirs)
+ create_elastic_env(params)
+ create_elastic_site(params, "elasticsearch.slave.yaml.j2")
+ create_elastic_config(params)
+ create_elastic_pam_limits(params)
+ if is_systemd_running():
+ configure_systemd(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
index 3e299e7..142ce4e 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_master.py
@@ -21,13 +21,11 @@ from resource_management.core import shell
from resource_management.core.resources.system import Execute
from resource_management.libraries.script import Script
from resource_management.core.logger import Logger
-
-from resource_management.core.exceptions import ExecutionFailed
-from resource_management.core.exceptions import ComponentIsNotRunning
-from elastic import elastic
-
+from elastic_commands import service_check
+from elastic_commands import configure_master
class Elasticsearch(Script):
+
def install(self, env):
import params
env.set_params(params)
@@ -38,57 +36,36 @@ class Elasticsearch(Script):
import params
env.set_params(params)
Logger.info('Configure Elasticsearch master node')
- elastic()
+ configure_master()
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
Logger.info('Stop Elasticsearch master node')
- stop_cmd = "service elasticsearch stop"
- Execute(stop_cmd)
+ Execute("service elasticsearch stop")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
Logger.info('Start Elasticsearch master node')
self.configure(env)
- start_cmd = "service elasticsearch start"
- Execute(start_cmd)
+ Execute("service elasticsearch start")
def status(self, env):
import params
env.set_params(params)
- Logger.info('Check status of Elasticsearch master node')
-
- # return codes defined by LSB
- # http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
- cmd = ('service', 'elasticsearch', 'status')
-
- rc, out = shell.call(cmd, sudo=True, quiet=False)
-
- if rc in [1, 2, 3]:
- # if return code = 1, 2, or 3, then 'program is not running' or 'dead'
- # Ambari's resource_management/libraries/script/script.py handles
- # this specific exception as OK
- Logger.info("Elasticsearch master is not running")
- raise ComponentIsNotRunning()
-
- elif rc == 0:
- # if return code = 0, then 'program is running or service is OK'
- Logger.info("Elasticsearch master is running")
-
- else:
- # else, program is dead or service state is unknown
- err_msg = "Execution of '{0}' returned {1}".format(" ".join(cmd), rc)
- raise ExecutionFailed(err_msg, rc, out)
+ Logger.info('Status check Elasticsearch master node')
+ service_check(
+ cmd="service elasticsearch status",
+ user=params.elastic_status_check_user,
+ label="Elasticsearch Master")
def restart(self, env):
import params
env.set_params(params)
- self.configure(env)
Logger.info('Restart Elasticsearch master node')
- restart_cmd = "service elasticsearch restart"
- Execute(restart_cmd)
+ self.configure(env)
+ Execute("service elasticsearch restart")
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
index d7684f0..2d559ff 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_slave.py
@@ -17,77 +17,54 @@ limitations under the License.
"""
-from resource_management.core import shell
-from resource_management.core.exceptions import ExecutionFailed
-from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Execute
from resource_management.libraries.script import Script
-from slave import slave
-
+from elastic_commands import service_check
+from elastic_commands import configure_slave
class Elasticsearch(Script):
+
def install(self, env):
import params
env.set_params(params)
- Logger.info('Install Elasticsearch data node')
+ Logger.info('Install Elasticsearch slave node')
self.install_packages(env)
def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
- Logger.info('Configure Elasticsearch data node')
- slave()
+ Logger.info('Configure Elasticsearch slave node')
+ configure_slave()
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
- Logger.info('Stop Elasticsearch data node')
- stop_cmd = "service elasticsearch stop"
- Execute(stop_cmd)
+ Logger.info('Stop Elasticsearch slave node')
+ Execute("service elasticsearch stop")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
self.configure(env)
- Logger.info('Start Elasticsearch data node')
- start_cmd = "service elasticsearch start"
- Execute(start_cmd)
+ Execute("service elasticsearch start")
def status(self, env):
import params
env.set_params(params)
- Logger.info('Check status of Elasticsearch data node')
-
- # return codes defined by LSB
- # http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
- cmd = ('service', 'elasticsearch', 'status')
- rc, out = shell.call(cmd, sudo=True, quiet=False)
-
- if rc in [1, 2, 3]:
- # if return code = 1, 2, or 3, then 'program is not running' or 'dead'
- # Ambari's resource_management/libraries/script/script.py handles
- # this specific exception as OK
- Logger.info("Elasticsearch slave is not running")
- raise ComponentIsNotRunning()
-
- elif rc == 0:
- # if return code = 0, then 'program is running or service is OK'
- Logger.info("Elasticsearch slave is running")
-
- else:
- # else, program is dead or service state is unknown
- err_msg = "Execution of '{0}' returned {1}".format(" ".join(cmd), rc)
- raise ExecutionFailed(err_msg, rc, out)
+ Logger.info('Status check Elasticsearch slave node')
+ service_check(
+ cmd="service elasticsearch status",
+ user=params.elastic_status_check_user,
+ label="Elasticsearch Slave")
def restart(self, env):
import params
env.set_params(params)
+ Logger.info('Restart Elasticsearch slave node')
self.configure(env)
- Logger.info('Restart Elasticsearch data node')
- restart_cmd = "service elasticsearch restart"
- Execute(restart_cmd)
+ Execute("service elasticsearch restart")
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
index 4adcf43..0399c60 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -90,3 +90,16 @@ elastic_user_nofile_limit = config['configurations']['elastic-env']['elastic_use
elastic_user_nproc_limit = config['configurations']['elastic-env']['elastic_user_nproc_limit']
elastic_user_memlock_soft_limit = config['configurations']['elastic-env']['elastic_user_memlock_soft_limit']
elastic_user_memlock_hard_limit = config['configurations']['elastic-env']['elastic_user_memlock_hard_limit']
+
+# the status check (service elasticsearch status) cannot be run by the 'elasticsearch'
+# user due to the default permissions that are set when the package is installed. the
+# status check must be run as root
+elastic_status_check_user = 'root'
+
+# when using the RPM or Debian packages on systems that use systemd, system limits
+# must be specified via systemd.
+# see https://www.elastic.co/guide/en/elasticsearch/reference/5.6/setting-system-settings.html#systemd
+systemd_parent_dir = '/etc/systemd/system/'
+systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
+systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
+systemd_override_template = config['configurations']['elastic-systemd']['content']
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
index 4f8b1ab..3ac7c83 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/service_check.py
@@ -17,6 +17,8 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+from __future__ import print_function
+
import subprocess
import sys
import re
@@ -26,32 +28,33 @@ from resource_management.libraries.script import Script
from resource_management.core.logger import Logger
class ServiceCheck(Script):
+
def service_check(self, env):
import params
env.set_params(params)
+ Logger.info("Running Elasticsearch service check")
- doc = '{"name": "Ambari Smoke test"}'
- index = "ambari_smoke_test"
+ port = self.get_port_from_range(params.http_port)
+ self.check_cluster_health(params.hostname, port)
+ self.index_document(params.hostname, port)
- # http_port from ES config may be a port range.
- es_http_port = re.search("^(\d+)", params.http_port).group(1)
- host = params.hostname + ":" + es_http_port
- Logger.info("Running Elastic search service check against " + host)
+ Logger.info("Elasticsearch service check successful")
+ exit(0)
- # Make sure the service is actually up. We can live without everything allocated.
- # Need both the retry and ES timeout. Can hit the URL before ES is ready at all and get no response, but can
- # also hit ES before things are green.
- Execute("curl -XGET 'http://%s/_cluster/health?wait_for_status=green&timeout=120s'" % host,
- logoutput=True,
- tries=6,
- try_sleep=20
- )
+ def index_document(self, host, port, doc='{"name": "Ambari Service Check"}', index="ambari_service_check"):
+ """
+ Tests the health of Elasticsearch by indexing a document.
- # Put a document into a new index.
- Execute("curl -XPUT '%s/%s/test/1' -d '%s'" % (host, index, doc), logoutput=True)
+ :param host: The name of a host running Elasticsearch.
+ :param port: The Elasticsearch HTTP port.
+ :param doc: The test document to put.
+ :param index: The name of the test index.
+ """
+ # put a document into a new index
+ Execute("curl -XPUT 'http://%s:%s/%s/test/1' -d '%s'" % (host, port, index, doc), logoutput=True)
- # Retrieve the document. Use subprocess because we actually need the results here.
- cmd_retrieve = "curl -XGET '%s/%s/test/1'" % (host, index)
+ # retrieve the document... use subprocess because we actually need the results here.
+ cmd_retrieve = "curl -XGET 'http://%s:%s/%s/test/1'" % (host, port, index)
proc = subprocess.Popen(cmd_retrieve, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
(stdout, stderr) = proc.communicate()
response_retrieve = stdout
@@ -59,8 +62,8 @@ class ServiceCheck(Script):
expected_retrieve = '{"_index":"%s","_type":"test","_id":"1","_version":1,"found":true,"_source":%s}' \
% (index, doc)
- # Delete the index
- cmd_delete = "curl -XDELETE '%s/%s'" % (host, index)
+ # delete the test index
+ cmd_delete = "curl -XDELETE 'http://%s:%s/%s'" % (host, port, index)
proc = subprocess.Popen(cmd_delete, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
(stdout, stderr) = proc.communicate()
response_delete = stdout
@@ -68,12 +71,43 @@ class ServiceCheck(Script):
expected_delete = '{"acknowledged":true}'
if (expected_retrieve == response_retrieve) and (expected_delete == response_delete):
- Logger.info("Smoke test able to communicate with Elasticsearch")
+ Logger.info("Successfully indexed document in Elasticsearch")
else:
- Logger.info("Elasticsearch service unable to retrieve document.")
+ Logger.info("Unable to retrieve document from Elasticsearch")
sys.exit(1)
- exit(0)
+ def check_cluster_health(self, host, port, status="green", timeout="120s"):
+ """
+ Checks Elasticsearch cluster health. Will wait for a given health
+ state to be reached.
+
+ :param host: The name of a host running Elasticsearch.
+ :param port: The Elasticsearch HTTP port.
+ :param status: The expected cluster health state. By default, green.
+ :param timeout: How long to wait for the cluster. By default, 120 seconds.
+ """
+ Logger.info("Checking cluster health")
+
+ cmd = "curl -sS -XGET 'http://{0}:{1}/_cluster/health?wait_for_status={2}&timeout={3}' | grep '\"status\":\"{2}\"'"
+ Execute(cmd.format(host, port, status, timeout), logoutput=True, tries=5, try_sleep=10)
+
+ def get_port_from_range(self, port_range, delimiter="-", default="9200"):
+ """
+ Elasticsearch is configured with a range of ports to bind to, such as
+ 9200-9300. This function identifies a single port within the given range.
+
+ :param port_range: A range of ports that Elasticsearch binds to.
+ :param delimiter: The port range delimiter, by default "-".
+ :param default: If no port can be identified in the port_range, the default is returned.
+ :return A single port within the given range.
+ """
+ port = default
+ if delimiter in port_range:
+ ports = port_range.split(delimiter)
+ if len(ports) > 0:
+ port = ports[0]
+
+ return port
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py
deleted file mode 100755
index a19989e..0000000
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/slave.py
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.core.resources.system import Directory
-from resource_management.core.resources.system import File
-from resource_management.core.source import InlineTemplate
-from resource_management.core.source import Template
-from resource_management.core.resources import User
-from resource_management.core.logger import Logger
-
-def slave():
- import params
-
- Logger.info("Creating user: {0}:{1}".format(params.elastic_user, params.elastic_group))
- User(params.elastic_user, action = "create", groups = params.elastic_group)
-
- params.path_data = params.path_data.replace('"', '')
- data_path = params.path_data.replace(' ', '').split(',')
- data_path[:] = [x.replace('"', '') for x in data_path]
- directories = [params.log_dir, params.pid_dir, params.conf_dir]
- directories = directories + data_path
-
- Logger.info("Creating directories: {0}".format(directories))
- Directory(directories,
- create_parents=True,
- mode=0755,
- owner=params.elastic_user,
- group=params.elastic_group,
- cd_access="a"
- )
-
- File("{0}/elastic-env.sh".format(params.conf_dir),
- owner=params.elastic_user,
- content=InlineTemplate(params.elastic_env_sh_template)
- )
-
- elastic_site = params.config['configurations']['elastic-site']
- path = "{0}/elasticsearch.yml".format(params.conf_dir)
- Logger.info("Creating ES slave configuration.")
- File(path,
- content=Template(
- "elasticsearch.slave.yaml.j2",
- configurations=elastic_site),
- owner=params.elastic_user,
- group=params.elastic_group
- )
-
- Logger.info("Slave sysconfig: /etc/sysconfig/elasticsearch")
- File(format("/etc/sysconfig/elasticsearch"),
- owner="root",
- group="root",
- content=InlineTemplate(params.sysconfig_template)
- )
-
- elastic_env = params.config['configurations']['elastic-env']
- Logger.info("Slave PAM limits: {0}".format(params.limits_conf_file))
- File(params.limits_conf_file,
- content=Template(
- 'elasticsearch_limits.conf.j2',
- configurations=elastic_env),
- owner="root",
- group="root"
- )
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
index 9cfb5cf..0629735 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/status_params.py
@@ -24,3 +24,4 @@ config = Script.get_config()
elastic_pid_dir = config['configurations']['elastic-env']['elastic_pid_dir']
elastic_pid_file = format("{elastic_pid_dir}/elasticsearch.pid")
+elastic_user = config['configurations']['elastic-env']['elastic_user']
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
index 034f71c..06b61a1 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/metainfo.xml
@@ -48,7 +48,7 @@
</components>
<osSpecifics>
<osSpecific>
- <osFamily>any</osFamily>
+ <osFamily>redhat6</osFamily>
<packages>
<package>
<name>python-elasticsearch</name>
@@ -58,6 +58,25 @@
</package>
</packages>
</osSpecific>
+ <osSpecific>
+ <osFamily>redhat7</osFamily>
+ <packages>
+ <package>
+ <name>python-elasticsearch</name>
+ </package>
+ <package>
+ <name>kibana-5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>ubuntu14</osFamily>
+ <packages>
+ <package>
+ <name>kibana=5.6.2</name>
+ </package>
+ </packages>
+ </osSpecific>
</osSpecifics>
<configuration-dependencies>
<config-type>kibana-env</config-type>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
new file mode 100644
index 0000000..37100cd
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/common.py
@@ -0,0 +1,56 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.logger import Logger
+from resource_management.libraries.functions.get_user_call_output import get_user_call_output
+from resource_management.core.exceptions import ExecutionFailed
+from resource_management.core.exceptions import ComponentIsNotRunning
+
+def service_check(cmd, user, label):
+ """
+ Executes a service check command that adheres to LSB-compliant
+ return codes. The return codes are interpreted as defined
+ by the LSB.
+
+ See http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
+ for more information.
+
+ :param cmd: The service check command to execute.
+ :param label: The name of the service.
+ """
+ Logger.info("Performing service check; cmd={0}, user={1}, label={2}".format(cmd, user, label))
+ rc, out, err = get_user_call_output(cmd, user, is_checked_call=False)
+
+ if len(err) > 0:
+ Logger.error(err)
+
+ if rc in [1, 2, 3]:
+ # if return code in [1, 2, 3], then 'program is not running' or 'program is dead'
+ Logger.info("{0} is not running".format(label))
+ raise ComponentIsNotRunning()
+
+ elif rc == 0:
+ # if return code = 0, then 'program is running or service is OK'
+ Logger.info("{0} is running".format(label))
+
+ else:
+ # else service state is unknown
+ err_msg = "{0} service check failed; cmd '{1}' returned {2}".format(label, cmd, rc)
+ Logger.error(err_msg)
+ raise ExecutionFailed(err_msg, rc, out, err)
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
index 861e19a..a15f709 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/KIBANA/5.6.2/package/scripts/kibana_master.py
@@ -14,18 +14,14 @@ distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-
-kibana_master
-
"""
import errno
import os
+from ambari_commons.os_check import OSCheck
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
-from resource_management.core import shell
-from resource_management.core.exceptions import ExecutionFailed
-from resource_management.core.exceptions import ComponentIsNotRunning
+
from resource_management.core.logger import Logger
from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import Execute
@@ -34,23 +30,23 @@ from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions.format import format as ambari_format
from resource_management.libraries.script import Script
+from common import service_check
class Kibana(Script):
+
def install(self, env):
import params
env.set_params(params)
- Logger.info("Install Kibana Master")
+ Logger.info("Installing Kibana")
self.install_packages(env)
def configure(self, env, upgrade_type=None, config_dir=None):
import params
env.set_params(params)
-
- Logger.info("Configure Kibana for Metron")
+ Logger.info("Configuring Kibana")
directories = [params.log_dir, params.pid_dir, params.conf_dir]
Directory(directories,
- create_parents=True,
mode=0755,
owner=params.kibana_user,
group=params.kibana_user
@@ -64,58 +60,28 @@ class Kibana(Script):
def stop(self, env, upgrade_type=None):
import params
env.set_params(params)
-
- Logger.info("Stop Kibana Master")
-
+ Logger.info("Stopping Kibana")
Execute("service kibana stop")
def start(self, env, upgrade_type=None):
import params
env.set_params(params)
-
self.configure(env)
-
- Logger.info("Start the Master")
-
-
+ Logger.info("Starting Kibana")
Execute("service kibana start")
def restart(self, env):
import params
env.set_params(params)
-
self.configure(env)
-
- Logger.info("Restarting the Master")
-
+ Logger.info("Restarting Kibana")
Execute("service kibana restart")
def status(self, env):
import params
env.set_params(params)
-
- Logger.info("Status of the Master")
-
- # return codes defined by LSB
- # http://refspecs.linuxbase.org/LSB_3.0.0/LSB-PDA/LSB-PDA/iniscrptact.html
- cmd = ('service', 'kibana', 'status')
- rc, out = shell.call(cmd, sudo=True, quiet=False)
-
- if rc in [1, 2, 3]:
- # if return code = 1, 2, or 3, then 'program is not running' or 'dead'
- # Ambari's resource_management/libraries/script/script.py handles
- # this specific exception as OK
- Logger.info("Kibana is not running")
- raise ComponentIsNotRunning()
-
- elif rc == 0:
- # if return code = 0, then 'program is running or service is OK'
- Logger.info("Kibana is running")
-
- else:
- # else, program is dead or service state is unknown
- err_msg = "Execution of '{0}' returned {1}".format(" ".join(cmd), rc)
- raise ExecutionFailed(err_msg, rc, out)
+ Logger.info('Status check Kibana')
+ service_check("service kibana status", user=params.kibana_user, label="Kibana")
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def load_template(self, env):
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
index 4b11456..18d5224 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_master.py
@@ -95,9 +95,8 @@ class Indexing(Script):
except Exception as e:
msg = "WARNING: Elasticsearch index templates could not be installed. " \
- "Is Elasticsearch running? error={0}"
+ "Is Elasticsearch running? Will reattempt install on next start. error={0}"
Logger.warning(msg.format(e))
- raise
commands.start_indexing_topology(env)
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/build.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/build.sh b/metron-deployment/packaging/docker/deb-docker/build.sh
index 0a79841..1fded4b 100755
--- a/metron-deployment/packaging/docker/deb-docker/build.sh
+++ b/metron-deployment/packaging/docker/deb-docker/build.sh
@@ -20,14 +20,11 @@ export FULL_VERSION=$1
export VERSION=$(echo ${FULL_VERSION} | tr -d '"'"'[:alpha:]'"'"')
export DISTRIBUTION="trusty"
-echo "FULL_VERSION: ${FULL_VERSION}"
-echo "VERSION: ${VERSION}"
-
INSTALL_PREFIX="/usr/metron"
METRON_HOME="${INSTALL_PREFIX}/${FULL_VERSION}"
HOMEDIR="/root"
WORKDIR="${HOMEDIR}/target"
-CONFIGDIR="${HOMEDIR}/debian/"
+CONFIGDIR="${HOMEDIR}/debian"
DATE=`date '+%Y%m%d%H%M'`
PREPACKAGEDIR="${HOMEDIR}/prepackage"
@@ -39,9 +36,26 @@ cd "${WORKDIR}"
for TARBALL in metron*.tar.gz; do
export PACKAGE=`echo ${TARBALL} | awk -F"-${FULL_VERSION}-" '{print $1}'`
+ if [ "${PACKAGE}" = "metron-pcap-backend" ]; then
+ # work around for inconsistent naming of 'metron-pcap-backend' ...
+ #
+ # * the tarball is named 'metron-pcap-backend'
+ # * but the RPM is created as 'metron-pcap'
+ # * the mpack also expects the package to be named 'metron-pcap'
+ #
+ # ... rather than use the tarball name here, rewrite the name to be
+ # consistent with the RPMs and MPack
+ export PACKAGE="metron-pcap"
+ fi
export PACKAGE_WORKDIR="${WORKDIR}/${PACKAGE}_${FULL_VERSION}"
echo "Building package; name=${PACKAGE}, tarball=${TARBALL}"
+ # if the tarball does not exist, fail fast
+ if [ ! -f "${TARBALL}" ]; then
+ echo "ERROR: Missing ${TARBALL}"
+ exit 1
+ fi
+
# extract the package contents
mkdir -p ${PACKAGE_WORKDIR}/${METRON_HOME}
tar xf ${TARBALL} -C ${PACKAGE_WORKDIR}/${METRON_HOME}
@@ -51,17 +65,36 @@ for TARBALL in metron*.tar.gz; do
PACKAGE_DEBIAN_DIR="${PACKAGE_WORKDIR}/DEBIAN"
mkdir ${PACKAGE_DEBIAN_DIR}
- # create the configuration files
- envsubst < ${CONFIGDIR}/control > ${PACKAGE_DEBIAN_DIR}/control
- envsubst < ${CONFIGDIR}/changelog > ${PACKAGE_DEBIAN_DIR}/changelog
- envsubst < ${CONFIGDIR}/copyright > ${PACKAGE_DEBIAN_DIR}/copyright
+ # all packages get the control files contained in `debian/metron`
+ for CFILE in ${CONFIGDIR}/metron/*; do
+ [ -e "$CFILE" ] || continue
+ CFILENAME=`basename "${CFILE}"`
+ DEST="${PACKAGE_DEBIAN_DIR}/${CFILENAME}"
+
+ # copy over the control file (allowing for variable substitution)
+ envsubst < ${CFILE} > ${DEST}
+
+ # strip comments from the control file
+ sed -i '/#[^!]*$/d' ${DEST}
+ done
+
+ # a package *may* have control files specific to it in `debian/$PACKAGE`
+ for CFILE in ${CONFIGDIR}/${PACKAGE}/*; do
+ [ -e "$CFILE" ] || continue
+ CFILENAME=`basename "${CFILE}"`
+ DEST="${PACKAGE_DEBIAN_DIR}/${CFILENAME}"
+
+ # copy over the control file (allowing for variable substitution)
+ envsubst < ${CFILE} > ${DEST}
+
+ # strip comments from the control file (don't delete shebangs!)
+ sed -i '/#[^!]*$/d' ${DEST}
- # strip comments from the config files
- sed -i 's/#.*$//g' ${PACKAGE_DEBIAN_DIR}/control
- sed -i 's/#.*$//g' ${PACKAGE_DEBIAN_DIR}/changelog
- sed -i 's/#.*$//g' ${PACKAGE_DEBIAN_DIR}/copyright
+ # permissions must be 0755 for maintain scripts like preinst and postinst
+ chmod 0755 ${DEST}
+ done
- # execute the prepackage script, if one exists
+ # execute the prepackage script, if one exists for the package
if [ -f "${PREPACKAGEDIR}/${PACKAGE}" ]; then
source ${PREPACKAGEDIR}/${PACKAGE}
fi
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/changelog
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/changelog b/metron-deployment/packaging/docker/deb-docker/debian/changelog
deleted file mode 100644
index 8b271bf..0000000
--- a/metron-deployment/packaging/docker/deb-docker/debian/changelog
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-$PACKAGE ($FULL_VERSION) $DISTRIBUTION; urgency=medium
-
- * Initial release. (Closes: METRON-1351)
-
- -- Apache Metron <de...@metron.apache.org> Wed, 13 Dec 2017 21:19:45 +0000
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/control
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/control b/metron-deployment/packaging/docker/deb-docker/debian/control
deleted file mode 100644
index febd3a9..0000000
--- a/metron-deployment/packaging/docker/deb-docker/debian/control
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Source: metron
-Section: misc
-Priority: extra
-Maintainer: Apache Metron <de...@metron.apache.org>
-Homepage: https://metron.apache.org/
-Package: $PACKAGE
-Architecture: all
-Version: $FULL_VERSION
-Depends:
-Description: Apache Metron
- Apache Metron provides a scalable advanced security analytics framework.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/copyright
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/copyright b/metron-deployment/packaging/docker/deb-docker/debian/copyright
deleted file mode 100644
index d02dfb0..0000000
--- a/metron-deployment/packaging/docker/deb-docker/debian/copyright
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Upstream-Name: $PACKAGE
-Upstream-Contact: Apache Metron <de...@metron.apache.org>
-
-Files: *
-License: ASL-2
-Copyright:
-Apache Metron
-Copyright 2015-2016 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-This product includes software developed by Cisco Systems (http://www.cisco.com)
-Copyright (c) 2014 Cisco Systems.
-
-This product includes software developed by Chef Software (https://www.chef.io)
-Copyright (c) 2012-2015, Chef Software, Inc.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postinst
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postinst
new file mode 100644
index 0000000..6f56bd7
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postinst
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed AFTER installation of the 'metron-alerts' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+# install the service script
+cp -f /usr/metron/${FULL_VERSION}/bin/metron-alerts-ui /etc/init.d/
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postrm
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postrm b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postrm
new file mode 100644
index 0000000..0ea1905
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/postrm
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed AFTER removal of the 'metron-alerts' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+rm -f /etc/init.d/metron-alerts-ui
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/preinst
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/preinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/preinst
new file mode 100644
index 0000000..33c7262
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/preinst
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed BEFORE installation of the 'metron-alerts' package.
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+if [ -f "/etc/init.d/metron-alerts-ui"]; then
+ # if service already exists, stop it before upgrading
+ /etc/init.d/metron-alerts-ui stop
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/prerm
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/prerm b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/prerm
new file mode 100644
index 0000000..403196b
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-alerts/prerm
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed BEFORE removal of the 'metron-alerts' package.
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+# if service exists, stop it
+if [ -f "/etc/init.d/metron-alerts-ui"]; then
+ /etc/init.d/metron-alerts-ui stop
+fi
+
+exit 0
[43/50] [abbrv] metron git commit: METRON-1444: Add Ubuntu
Repositories for Elasticsearch to the Mpack (mmiklavc via mmiklavc) closes
apache/metron#928
Posted by rm...@apache.org.
METRON-1444: Add Ubuntu Repositories for Elasticsearch to the Mpack (mmiklavc via mmiklavc) closes apache/metron#928
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/48d9d256
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/48d9d256
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/48d9d256
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 48d9d25696a71ff07791b86348dc978db7a56a9d
Parents: e5f24f7
Author: mmiklavc <mi...@gmail.com>
Authored: Thu Feb 8 11:53:47 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Thu Feb 8 11:53:47 2018 -0700
----------------------------------------------------------------------
.../roles/ambari_common/defaults/main.yml | 6 +--
.../ambari_common/tasks/elasticsearch-repo.yml | 40 --------------------
.../ansible/roles/ambari_common/tasks/main.yml | 1 -
.../packaging/ambari/metron-mpack/README.md | 14 -------
.../ELASTICSEARCH/5.6.2/repos/repoinfo.xml | 13 +++++++
.../KIBANA/5.6.2/repos/repoinfo.xml | 19 ++++++++++
6 files changed, 34 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
index 0614e0f..e7968f5 100644
--- a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
@@ -17,7 +17,5 @@
---
hadoop_logrotate_frequency: daily
hadoop_logrotate_retention: 30
-centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.5.2.0/ambari.repo
-ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.5.2.0
-ubuntu_elasticsearch_packages_repo: https://artifacts.elastic.co/packages/5.x/apt
-ubuntu_elasticsearch_curator_repo: https://packages.elastic.co/curator/5/debian
+centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.6.1.0/ambari.repo
+ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.6.1.0
http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml b/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
deleted file mode 100644
index 730607d..0000000
--- a/metron-deployment/ansible/roles/ambari_common/tasks/elasticsearch-repo.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-#
-# for CentOS...
-#
-# this is not needed for CentOS. the elasticsearch repository is automatically
-# added by the mpack. there is currently an issue that needs resolved that
-# prevents us from doing the same in Ubuntu
-#
-
-#
-# for Ubuntu...
-#
-- name: Setup Elasticsearch packages repo on Ubuntu
- shell: echo "deb {{ ubuntu_elasticsearch_packages_repo }} stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch.list
- when: ansible_distribution == "Ubuntu"
-
-- name: Setup Elasticsearch curator repo on Ubuntu
- shell: echo "deb {{ ubuntu_elasticsearch_curator_repo }} stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch.list
- when: ansible_distribution == "Ubuntu"
-
-- name: Update package cache
- apt: update_cache=yes
- when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
index b3258a9..f3fd905 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
@@ -34,6 +34,5 @@
static: no
when: ansible_distribution == "Ubuntu"
-- include: elasticsearch-repo.yml
- include: nodejs.yml
- include: logrotate.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/packaging/ambari/metron-mpack/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md b/metron-deployment/packaging/ambari/metron-mpack/README.md
index 4b0b073..5179b5f 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/README.md
+++ b/metron-deployment/packaging/ambari/metron-mpack/README.md
@@ -28,20 +28,6 @@ This allows you to easily install Metron using a simple, guided process. This a
* A [Node.js](https://nodejs.org/en/download/package-manager/) repository installed on the host running the Management and Alarm UI.
-* When installing on Ubuntu the Elasticsearch repository must be defined manually. This is NOT defined by the Mpack like it is on CentOS. This is an open bug that needs addressed in the Mpack. See the [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html) documentation for more specific instructions.
- ```
- $ cat >/etc/apt/sources.list.d/elasticsearch.list << EOL
- deb https://packages.elastic.co/curator/5/debian stable main
- deb https://artifacts.elastic.co/packages/5.x/apt stable main
- EOL
-
- $ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
-
- $ apt-get update
- ```
-
-
-
### Quick Start
1. Build the Metron MPack. Execute the following command from the project's root directory.
http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
index 2c64518..ba21fb1 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/ELASTICSEARCH/5.6.2/repos/repoinfo.xml
@@ -28,5 +28,18 @@
<reponame>ELASTICSEARCH</reponame>
</repo>
</os>
+ <os family="ubuntu14">
+ <!--
+ see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
+ https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
+ https://issues.apache.org/jira/browse/AMBARI-21856
+ -->
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
+ <repoid>elasticsearch-5.x</repoid>
+ <reponame>ELASTICSEARCH</reponame>
+ <distribution>stable</distribution>
+ </repo>
+ </os>
</reposinfo>
http://git-wip-us.apache.org/repos/asf/metron/blob/48d9d256/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
index d0bd284..2755818 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/addon-services/KIBANA/5.6.2/repos/repoinfo.xml
@@ -38,4 +38,23 @@
<reponame>CURATOR</reponame>
</repo>
</os>
+ <os family="ubuntu14">
+ <!--
+ see details about Ambari fixes for Ubuntu introduced in Ambari 2.6+
+ https://github.com/apache/ambari/commit/f8b29df9685b443d4a5c06c6e1725e4428c95b49#diff-6f26c26ed59462200d018c5e1e71e773
+ https://issues.apache.org/jira/browse/AMBARI-21856
+ -->
+ <repo>
+ <baseurl>https://artifacts.elastic.co/packages/5.x/apt</baseurl>
+ <repoid>kibana-5.x</repoid>
+ <reponame>KIBANA</reponame>
+ <distribution>stable</distribution>
+ </repo>
+ <repo>
+ <baseurl>https://packages.elastic.co/curator/5/debian</baseurl>
+ <repoid>ES-Curator-5.x</repoid>
+ <reponame>CURATOR</reponame>
+ <distribution>stable</distribution>
+ </repo>
+ </os>
</reposinfo>
[21/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/files/yaf.out
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/files/yaf.out b/metron-deployment/ansible/roles/sensor-stubs/files/yaf.out
new file mode 100644
index 0000000..b7c4710
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/files/yaf.out
@@ -0,0 +1,22164 @@
+2017-01-11 20:52:24.012|2017-01-11 20:52:24.012| 0.000| 0.000| 6| 192.168.138.158|49195| 188.165.164.184| 80| AR| 0| 0| 0|522c999d|00000000|000|000| 1| 40| 0| 0| 0|
+2017-01-11 20:52:24.008|2017-01-11 20:52:24.022| 0.014| 0.002| 6| 72.34.49.86| 80| 192.168.138.158|49204| AP| APF| AF| A|3e05e129|5869e531|000|000| 3| 358| 2| 80| 0|
+2017-01-11 20:52:36.640|2017-01-11 20:53:28.138| 51.498| 0.000| 6| 62.75.195.236| 80| 192.168.138.158|49186| AP| AP| AR| A|73734c1c|516c4a0c|000|000| 2| 82| 3| 120| 0|
+2017-01-11 20:52:52.845|2017-01-11 20:53:58.724| 65.879| 0.187| 6| 192.168.138.158|49208| 95.163.121.204| 80| S| APR| AS| AP|be149584|8ef40c65|000|000| 7| 1206| 6| 2873| 0|
+2017-01-11 20:52:52.845|2017-01-11 20:54:03.711| 70.866| 0.187| 6| 192.168.138.158|49207| 95.163.121.204| 80| S| APR| AS| AP|c031307b|d1fe0059|000|000| 7| 1088| 5| 1685| 0|
+2017-01-11 20:52:52.845|2017-01-11 20:54:08.699| 75.854| 0.201| 6| 192.168.138.158|49209| 95.163.121.204| 80| S| APR| AS| AP|32c0b55e|f31124b8|000|000| 8| 1502| 9| 6752| 0|
+2017-01-11 20:52:50.865|2017-01-11 20:54:08.699| 77.834| 0.173| 6| 192.168.138.158|49206| 95.163.121.204| 80| S| APR| AS| AP|a80daa6e|b93a04df|000|000| 10| 2265| 12| 7194| 0|
+2017-01-11 20:52:48.208|2017-01-11 20:54:08.700| 80.492| 0.172| 6| 192.168.138.158|49205| 95.163.121.204| 80| S| APR| AS| AP|ca8d81e2|0628e3a4|000|000| 11| 2097| 14| 9627| 0|
+2017-01-11 20:52:52.847|2017-01-11 20:54:08.700| 75.853| 0.185| 6| 192.168.138.158|49210| 95.163.121.204| 80| S| APR| AS| AP|63626a70|9b7a5687|000|000| 8| 1645| 7| 2238| 0|
+2017-01-11 20:54:08.935|2017-01-11 20:54:09.448| 0.513| 0.132| 6| 192.168.138.158|49184| 62.75.195.236| 80| S| APF| AS| APF|92a7a033|00b98442|000|000| 5| 805| 5| 966| 0|
+2017-01-11 20:54:09.637|2017-01-11 20:54:10.054| 0.417| 0.131| 6| 192.168.138.158|49185| 62.75.195.236| 80| S| APF| AS| APF|14dad2de|057d07a2|000|000| 6| 782| 10| 9600| 0|
+2017-01-11 20:54:10.747|2017-01-11 20:54:11.162| 0.415| 0.136| 6| 192.168.138.158|49188| 62.75.195.236| 80| S| APF| AS| APF|f017c491|abdb7ffd|000|000| 5| 283| 5| 1267| 0|
+2017-01-11 20:54:11.153|2017-01-11 20:54:12.135| 0.982| 0.135| 6| 192.168.138.158|49189| 62.75.195.236| 80| S| APF| AS| APF|9dfb17ed|f1bbe6ec|000|000| 80| 3525| 167| 229440| 0|
+2017-01-11 20:54:13.341|2017-01-11 20:54:13.628| 0.287| 0.131| 6| 192.168.138.158|49190| 62.75.195.236| 80| S| APF| AS| APF|abe3db05|2178457a|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:54:13.630|2017-01-11 20:54:13.941| 0.311| 0.153| 6| 192.168.138.158|49191| 62.75.195.236| 80| S| APF| AS| APF|6ac7f41d|1ecaba70|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:54:13.943|2017-01-11 20:54:14.480| 0.537| 0.155| 6| 192.168.138.158|49192| 62.75.195.236| 80| S| APF| AS| APF|d57e1fff|00b640f3|000|000| 5| 525| 5| 418| 0|
+2017-01-11 20:54:14.483|2017-01-11 20:54:14.793| 0.310| 0.149| 6| 192.168.138.158|49193| 62.75.195.236| 80| S| APF| AS| APF|1cff6791|0099d801|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:54:14.795|2017-01-11 20:54:15.093| 0.298| 0.140| 6| 192.168.138.158|49194| 62.75.195.236| 80| S| APF| AS| APF|d7003ea2|c8807b47|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:54:15.133|2017-01-11 20:54:15.334| 0.201| 0.042| 6| 192.168.138.158|49197| 204.152.254.221| 80| S| APF| AS| APF|58cf2c67|166f1d89|000|000| 7| 820| 6| 765| 0|
+2017-01-11 20:54:15.094|2017-01-11 20:54:15.388| 0.294| 0.138| 6| 192.168.138.158|49196| 62.75.195.236| 80| S| APF| AS| APF|bdd9bac5|546ac731|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:54:15.555|2017-01-11 20:54:17.711| 2.156| 0.051| 6| 192.168.138.158|49198| 72.34.49.86| 80| S| APF| AS| APF|24718a4a|e263a145|000|000| 7| 822| 7| 522| 0|
+2017-01-11 20:54:17.700|2017-01-11 20:54:17.886| 0.186| 0.059| 6| 192.168.138.158|49199| 204.152.254.221| 80| S| APF| AS| APF|86fa40fb|20de0881|000|000| 6| 741| 5| 725| 0|
+2017-01-11 20:54:17.889|2017-01-11 20:54:20.677| 2.788| 0.103| 6| 192.168.138.158|49200| 72.34.49.86| 80| S| APF| AS| APF|788e6fcc|271a3fd2|000|000| 6| 743| 6| 1466| 0|
+2017-01-11 20:54:20.666|2017-01-11 20:54:20.832| 0.166| 0.053| 6| 192.168.138.158|49201| 204.152.254.221| 80| S| APF| AS| APF|b74faf85|50818fd9|000|000| 7| 846| 6| 765| 0|
+2017-01-11 20:54:20.834|2017-01-11 20:54:24.431| 3.597| 0.106| 6| 192.168.138.158|49202| 72.34.49.86| 80| S| APF| AS| APF|78494ef3|0d188aea|000|000| 19| 1328| 39| 47453| 0|
+2017-01-11 20:54:27.057|2017-01-11 20:54:27.252| 0.195| 0.043| 6| 192.168.138.158|49203| 204.152.254.221| 80| S| APF| AS| APF|075be34c|00357320|000|000| 6| 754| 5| 725| 0|
+2017-01-11 20:54:14.819|2017-01-11 20:54:29.238| 14.419| 0.129| 6| 192.168.138.158|49195| 188.165.164.184| 80| S| APR| AS| AP|522c98b3|c88832bc|000|000| 5| 445| 3| 552| 0|
+2017-01-11 20:54:27.255|2017-01-11 20:54:29.248| 1.993| 0.105| 6| 192.168.138.158|49204| 72.34.49.86| 80| S| APF| AS| APF|5869e338|3e05e128|000|000| 6| 756| 6| 482| 0|
+2017-01-11 20:54:09.651|2017-01-11 20:55:33.365| 83.714| 0.131| 6| 192.168.138.158|49186| 62.75.195.236| 80| S| APR| AS| AP|516c475c|7371702c|000|000| 51| 2739| 100| 125851| 0|
+2017-01-11 20:54:58.071|2017-01-11 20:56:03.951| 65.880| 0.187| 6| 192.168.138.158|49208| 95.163.121.204| 80| S| APR| AS| AP|be149584|8ef40c65|000|000| 7| 1206| 6| 2873| 0|
+2017-01-11 20:54:58.070|2017-01-11 20:56:08.937| 70.867| 0.188| 6| 192.168.138.158|49207| 95.163.121.204| 80| S| APR| AS| AP|c031307b|d1fe0059|000|000| 7| 1088| 5| 1685| 0|
+2017-01-11 20:54:58.071|2017-01-11 20:56:13.926| 75.855| 0.201| 6| 192.168.138.158|49209| 95.163.121.204| 80| S| APR| AS| AP|32c0b55e|f31124b8|000|000| 8| 1502| 9| 6752| 0|
+2017-01-11 20:54:56.089|2017-01-11 20:56:13.926| 77.837| 0.174| 6| 192.168.138.158|49206| 95.163.121.204| 80| S| APR| AS| AP|a80daa6e|b93a04df|000|000| 10| 2265| 12| 7194| 0|
+2017-01-11 20:54:53.434|2017-01-11 20:56:13.926| 80.492| 0.173| 6| 192.168.138.158|49205| 95.163.121.204| 80| S| APR| AS| AP|ca8d81e2|0628e3a4|000|000| 11| 2097| 14| 9627| 0|
+2017-01-11 20:54:58.073|2017-01-11 20:56:13.926| 75.853| 0.185| 6| 192.168.138.158|49210| 95.163.121.204| 80| S| APR| AS| AP|63626a70|9b7a5687|000|000| 8| 1645| 7| 2238| 0|
+2017-01-11 20:56:14.161|2017-01-11 20:56:14.672| 0.511| 0.133| 6| 192.168.138.158|49184| 62.75.195.236| 80| S| APF| AS| APF|92a7a033|00b98442|000|000| 5| 805| 5| 966| 0|
+2017-01-11 20:56:14.861|2017-01-11 20:56:15.278| 0.417| 0.132| 6| 192.168.138.158|49185| 62.75.195.236| 80| S| APF| AS| APF|14dad2de|057d07a2|000|000| 6| 782| 10| 9600| 0|
+2017-01-11 20:56:15.960|2017-01-11 20:56:16.373| 0.413| 0.136| 6| 192.168.138.158|49188| 62.75.195.236| 80| S| APF| AS| APF|f017c491|abdb7ffd|000|000| 5| 283| 5| 1267| 0|
+2017-01-11 20:56:16.364|2017-01-11 20:56:17.370| 1.006| 0.135| 6| 192.168.138.158|49189| 62.75.195.236| 80| S| APF| AS| APF|9dfb17ed|f1bbe6ec|000|000| 80| 3525| 167| 229440| 0|
+2017-01-11 20:56:18.575|2017-01-11 20:56:18.864| 0.289| 0.131| 6| 192.168.138.158|49190| 62.75.195.236| 80| S| APF| AS| APF|abe3db05|2178457a|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:56:18.865|2017-01-11 20:56:19.176| 0.311| 0.153| 6| 192.168.138.158|49191| 62.75.195.236| 80| S| APF| AS| APF|6ac7f41d|1ecaba70|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:56:19.177|2017-01-11 20:56:19.712| 0.535| 0.154| 6| 192.168.138.158|49192| 62.75.195.236| 80| S| APF| AS| APF|d57e1fff|00b640f3|000|000| 5| 525| 5| 418| 0|
+2017-01-11 20:56:19.715|2017-01-11 20:56:20.025| 0.310| 0.150| 6| 192.168.138.158|49193| 62.75.195.236| 80| S| APF| AS| APF|1cff6791|0099d801|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:56:20.027|2017-01-11 20:56:20.325| 0.298| 0.140| 6| 192.168.138.158|49194| 62.75.195.236| 80| S| APF| AS| APF|d7003ea2|c8807b47|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:56:20.365|2017-01-11 20:56:20.567| 0.202| 0.042| 6| 192.168.138.158|49197| 204.152.254.221| 80| S| APF| AS| APF|58cf2c67|166f1d89|000|000| 7| 820| 6| 765| 0|
+2017-01-11 20:56:20.326|2017-01-11 20:56:20.620| 0.294| 0.139| 6| 192.168.138.158|49196| 62.75.195.236| 80| S| APF| AS| APF|bdd9bac5|546ac731|000|000| 5| 525| 4| 378| 0|
+2017-01-11 20:56:20.787|2017-01-11 20:56:22.941| 2.154| 0.051| 6| 192.168.138.158|49198| 72.34.49.86| 80| S| APF| AS| APF|24718a4a|e263a145|000|000| 7| 822| 7| 522| 0|
+2017-01-11 20:56:22.930|2017-01-11 20:56:23.114| 0.184| 0.059| 6| 192.168.138.158|49199| 204.152.254.221| 80| S| APF| AS| APF|86fa40fb|20de0881|000|000| 6| 741| 5| 725| 0|
+2017-01-11 20:56:23.118|2017-01-11 20:56:25.907| 2.789| 0.105| 6| 192.168.138.158|49200| 72.34.49.86| 80| S| APF| AS| APF|788e6fcc|271a3fd2|000|000| 6| 743| 6| 1466| 0|
+2017-01-11 20:56:25.896|2017-01-11 20:56:26.062| 0.166| 0.054| 6| 192.168.138.158|49201| 204.152.254.221| 80| S| APF| AS| APF|b74faf85|50818fd9|000|000| 7| 846| 6| 765| 0|
+2017-01-11 20:56:26.064|2017-01-11 20:56:29.665| 3.601| 0.106| 6| 192.168.138.158|49202| 72.34.49.86| 80| S| APF| AS| APF|78494ef3|0d188aea|000|000| 19| 1328| 39| 47453| 0|
+2017-01-11 20:56:32.291|2017-01-11 20:56:32.484| 0.193| 0.042| 6| 192.168.138.158|49203| 204.152.254.221| 80| S| APF| AS| APF|075be34c|00357320|000|000| 6| 754| 5| 725| 0|
+2017-01-11 20:56:20.052|2017-01-11 20:56:34.469| 14.417| 0.128| 6| 192.168.138.158|49195| 188.165.164.184| 80| S| APR| AS| AP|522c98b3|c88832bc|000|000| 5| 445| 3| 552| 0|
+2017-01-11 20:56:32.486|2017-01-11 20:56:34.479| 1.993| 0.105| 6| 192.168.138.158|49204| 72.34.49.86| 80| S| APF| AS| APF|5869e338|3e05e128|000|000| 6| 756| 6| 482| 0|
+2017-01-11 20:53:15.474|2017-01-11 20:56:10.859| 175.385| 0.000| 17| 192.168.66.1| 5353| 224.0.0.251| 5353| 0| 0| 0| 0|00000000|00000000|000|000| 12| 816| 0| 0| 0|eof
+2017-01-11 20:54:08.700|2017-01-11 20:56:14.077| 125.377| 0.151| 17| 192.168.138.158|60078| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 246| 2| 278| 0|eof
+2017-01-11 20:54:09.492|2017-01-11 20:56:14.861| 125.369| 0.144| 17| 192.168.138.158|65315| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 244| 2| 276| 0|eof
+2017-01-11 20:54:09.492|2017-01-11 20:56:14.875| 125.383| 0.158| 17| 192.168.138.158|50683| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 242| 2| 274| 0|eof
+2017-01-11 20:52:23.006|2017-01-11 20:56:17.746| 234.740| 0.022| 6| 192.168.66.1|50181| 192.168.66.121| 8080| AP| AP| AP| AP|89de8804|413104f3|000|000| 20| 6226| 14| 6653| 0|eof
+2017-01-11 20:54:14.677|2017-01-11 20:56:20.051| 125.374| 0.142| 17| 192.168.138.158|53571| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 112| 2| 144| 0|eof
+2017-01-11 20:54:15.095|2017-01-11 20:56:20.364| 125.269| 0.037| 17| 192.168.138.158|61720| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 112| 2| 144| 0|eof
+2017-01-11 20:54:15.336|2017-01-11 20:56:20.753| 125.417| 0.185| 17| 192.168.138.158|50509| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 150| 2| 262| 0|eof
+2017-01-11 20:54:15.522|2017-01-11 20:56:20.786| 125.264| 0.033| 17| 192.168.138.158|56753| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 2| 128| 2| 160| 0|eof
+2017-01-11 20:52:23.007|2017-01-11 20:56:24.203| 241.196| 0.039| 6| 192.168.66.1|50186| 192.168.66.121| 8080| AP| AP| AP| AP|e6b20f8c|79c1af21|000|000| 75| 34066| 56| 30255| 0|eof
+2017-01-11 20:56:14.875|2017-01-11 20:56:47.098| 32.223| 0.131| 6| 192.168.138.158|49186| 62.75.195.236| 80| S| AP| AS| AP|516c475c|7371702c|000|000| 49| 2659| 99| 125810| 0|eof
+2017-01-11 20:52:23.007|2017-01-11 20:56:52.666| 269.659| 0.011| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| AP| AP| AP|7b810bc7|f502ff81|000|000| 76| 28692| 56| 30587| 0|eof
+2017-01-11 20:52:23.005|2017-01-11 20:56:52.686| 269.681| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| AP| AP| AP|e53b601f|185bd4f0|000|000| 96| 38685| 70| 39312| 0|eof
+2017-01-11 20:52:23.007|2017-01-11 20:56:53.683| 270.676| 0.040| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| AP| AP| AP|3982f0c2|a885b507|000|000| 235| 95979| 174| 101932| 0|eof
+2017-01-11 20:52:23.006|2017-01-11 20:56:56.041| 273.035| 0.008| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| AP| AP| AP|83665d75|8df32014|000|000| 332| 146689| 257| 146440| 0|eof
+2017-01-11 20:52:48.175|2017-01-11 20:56:58.664| 250.489| 0.032| 17| 192.168.138.158|50329| 192.168.138.2| 53| 0| 0| 0| 0|00000000|00000000|000|000| 3| 231| 3| 279| 0|eof
+2017-01-11 20:56:58.838|2017-01-11 20:56:58.838| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AS| 0| 0| 0|0628e3a4|00000000|000|000| 1| 44| 0| 0| 0|idle
+2017-01-11 20:56:58.839|2017-01-11 20:56:58.839| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d81e3|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:56:58.840|2017-01-11 20:56:58.840| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d81e3|00000000|000|000| 1| 369| 0| 0| 0|idle
+2017-01-11 20:56:58.840|2017-01-11 20:56:58.840| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628e3a5|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:56:59.992|2017-01-11 20:56:59.992| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83685706|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:56:59.992|2017-01-11 20:56:59.992| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83685cae|00000000|000|000| 1| 908| 0| 0| 0|idle
+2017-01-11 20:56:59.992|2017-01-11 20:56:59.992| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df527e8|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|398437fd|00000000|000|000| 1| 864| 0| 0| 0|idle
+2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53be3e0|00000000|000|000| 1| 580| 0| 0| 0|idle
+2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6048|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b816c8f|00000000|000|000| 1| 629| 0| 0| 0|idle
+2017-01-11 20:56:59.993|2017-01-11 20:56:59.993| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| A| 0| 0| 0|f5036b9c|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:56:59.995|2017-01-11 20:56:59.995| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6048|00000000|000|000| 1| 302| 0| 0| 0|idle
+2017-01-11 20:56:59.995|2017-01-11 20:56:59.995| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53be5f0|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:00.010|2017-01-11 20:57:00.010| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f5036b9c|00000000|000|000| 1| 1032| 0| 0| 0|idle
+2017-01-11 20:57:00.010|2017-01-11 20:57:00.010| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b816ed0|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:00.032|2017-01-11 20:57:00.032| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8871fdb|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:00.069|2017-01-11 20:57:00.069| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df527e8|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:00.069|2017-01-11 20:57:00.069| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df52d90|00000000|000|000| 1| 189| 0| 0| 0|idle
+2017-01-11 20:57:00.069|2017-01-11 20:57:00.069| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686006|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:00.095|2017-01-11 20:57:00.095| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8871fdb|00000000|000|000| 1| 1067| 0| 0| 0|idle
+2017-01-11 20:57:00.095|2017-01-11 20:57:00.095| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843b29|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.317|2017-01-11 20:57:01.317| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628e3a5|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:01.322|2017-01-11 20:57:01.322| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| S| 0| 0| 0|a80daa6e|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.322|2017-01-11 20:57:01.322| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628e8f0|00000000|000|000| 1| 393| 0| 0| 0|idle
+2017-01-11 20:57:01.323|2017-01-11 20:57:01.323| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d832c|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:01.495|2017-01-11 20:57:01.495| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AS| 0| 0| 0|b93a04df|00000000|000|000| 1| 44| 0| 0| 0|idle
+2017-01-11 20:57:01.495|2017-01-11 20:57:01.495| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80daa6f|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:01.496|2017-01-11 20:57:01.496| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80daa6f|00000000|000|000| 1| 478| 0| 0| 0|idle
+2017-01-11 20:57:01.496|2017-01-11 20:57:01.496| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a04e0|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:01.942|2017-01-11 20:57:01.942| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|39843b29|00000000|000|000| 1| 504| 0| 0| 0|idle
+2017-01-11 20:57:01.942|2017-01-11 20:57:01.942| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a88723d2|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.943|2017-01-11 20:57:01.943| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686006|00000000|000|000| 1| 628| 0| 0| 0|idle
+2017-01-11 20:57:01.943|2017-01-11 20:57:01.943| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b816ed0|00000000|000|000| 1| 678| 0| 0| 0|idle
+2017-01-11 20:57:01.948|2017-01-11 20:57:01.948| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df52e19|00000000|000|000| 1| 502| 0| 0| 0|idle
+2017-01-11 20:57:01.948|2017-01-11 20:57:01.948| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686246|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.950|2017-01-11 20:57:01.950| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f5036f70|00000000|000|000| 1| 544| 0| 0| 0|idle
+2017-01-11 20:57:01.950|2017-01-11 20:57:01.950| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817142|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a88723d2|00000000|000|000| 1| 2948| 0| 0| 0|idle
+2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8872f22|00000000|000|000| 1| 151| 0| 0| 0|idle
+2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843ced|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.959|2017-01-11 20:57:01.959| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843ced|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:01.990|2017-01-11 20:57:01.990| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843ced|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:03.212|2017-01-11 20:57:03.212| 0.000| 0.000| 6| 192.168.66.1|50181| 192.168.66.121| 8080| A| 0| 0| 0|89de9c75|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.212|2017-01-11 20:57:03.212| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50181| A| 0| 0| 0|41311c18|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:03.281|2017-01-11 20:57:03.281| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a04e0|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:03.295|2017-01-11 20:57:03.295| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a0a2b|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:03.295|2017-01-11 20:57:03.295| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a0f76|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:03.295|2017-01-11 20:57:03.295| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a152a|00000000|000|000| 1| 636| 0| 0| 0|idle
+2017-01-11 20:57:03.296|2017-01-11 20:57:03.296| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80dac25|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d832c|00000000|000|000| 1| 481| 0| 0| 0|idle
+2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628ea51|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80dac25|00000000|000|000| 1| 481| 0| 0| 0|idle
+2017-01-11 20:57:03.301|2017-01-11 20:57:03.301| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a177e|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.302|2017-01-11 20:57:03.302| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| S| 0| 0| 0|c031307b|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:03.302|2017-01-11 20:57:03.302| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| S| 0| 0| 0|be149584|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:03.302|2017-01-11 20:57:03.302| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| S| 0| 0| 0|32c0b55e|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:03.304|2017-01-11 20:57:03.304| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| S| 0| 0| 0|63626a70|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:03.489|2017-01-11 20:57:03.489| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AS| 0| 0| 0|8ef40c65|00000000|000|000| 1| 44| 0| 0| 0|idle
+2017-01-11 20:57:03.489|2017-01-11 20:57:03.489| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| AS| 0| 0| 0|d1fe0059|00000000|000|000| 1| 44| 0| 0| 0|idle
+2017-01-11 20:57:03.489|2017-01-11 20:57:03.489| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AS| 0| 0| 0|9b7a5687|00000000|000|000| 1| 44| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| A| 0| 0| 0|be149585|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| A| 0| 0| 0|c031307c|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| A| 0| 0| 0|63626a71|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| AP| 0| 0| 0|be149585|00000000|000|000| 1| 519| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| A| 0| 0| 0|8ef40c66|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| AP| 0| 0| 0|c031307c|00000000|000|000| 1| 481| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| A| 0| 0| 0|d1fe005a|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.490|2017-01-11 20:57:03.490| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| AP| 0| 0| 0|63626a71|00000000|000|000| 1| 475| 0| 0| 0|idle
+2017-01-11 20:57:03.491|2017-01-11 20:57:03.491| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| A| 0| 0| 0|9b7a5688|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.503|2017-01-11 20:57:03.503| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AS| 0| 0| 0|f31124b8|00000000|000|000| 1| 44| 0| 0| 0|idle
+2017-01-11 20:57:03.503|2017-01-11 20:57:03.503| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b55f|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:03.506|2017-01-11 20:57:03.506| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| AP| 0| 0| 0|32c0b55f|00000000|000|000| 1| 481| 0| 0| 0|idle
+2017-01-11 20:57:03.506|2017-01-11 20:57:03.506| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| A| 0| 0| 0|f31124b9|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.068|2017-01-11 20:57:05.068| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a177e|00000000|000|000| 1| 856| 0| 0| 0|idle
+2017-01-11 20:57:05.070|2017-01-11 20:57:05.070| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80dadde|00000000|000|000| 1| 481| 0| 0| 0|idle
+2017-01-11 20:57:05.071|2017-01-11 20:57:05.071| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a1aae|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.283|2017-01-11 20:57:05.283| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628ea51|00000000|000|000| 1| 1129| 0| 0| 0|idle
+2017-01-11 20:57:05.286|2017-01-11 20:57:05.286| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d84e5|00000000|000|000| 1| 475| 0| 0| 0|idle
+2017-01-11 20:57:05.286|2017-01-11 20:57:05.286| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628ee92|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.319|2017-01-11 20:57:05.319| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AP| 0| 0| 0|9b7a5688|00000000|000|000| 1| 529| 0| 0| 0|idle
+2017-01-11 20:57:05.321|2017-01-11 20:57:05.321| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| AP| 0| 0| 0|63626c24|00000000|000|000| 1| 475| 0| 0| 0|idle
+2017-01-11 20:57:05.322|2017-01-11 20:57:05.322| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| A| 0| 0| 0|9b7a5871|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.403|2017-01-11 20:57:05.403| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AP| 0| 0| 0|8ef40c66|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:05.418|2017-01-11 20:57:05.418| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AP| 0| 0| 0|8ef411b1|00000000|000|000| 1| 828| 0| 0| 0|idle
+2017-01-11 20:57:05.418|2017-01-11 20:57:05.418| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| A| 0| 0| 0|be149764|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.419|2017-01-11 20:57:05.419| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| AP| 0| 0| 0|be149764|00000000|000|000| 1| 475| 0| 0| 0|idle
+2017-01-11 20:57:05.419|2017-01-11 20:57:05.419| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| A| 0| 0| 0|8ef414c5|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.557|2017-01-11 20:57:05.557| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| AP| 0| 0| 0|d1fe005a|00000000|000|000| 1| 938| 0| 0| 0|idle
+2017-01-11 20:57:05.658|2017-01-11 20:57:05.658| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| A| 0| 0| 0|c0313235|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:05.682|2017-01-11 20:57:05.682| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f31124b9|00000000|000|000| 1| 838| 0| 0| 0|idle
+2017-01-11 20:57:05.785|2017-01-11 20:57:05.785| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b718|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:07.100|2017-01-11 20:57:07.100| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|39843ced|00000000|000|000| 1| 580| 0| 0| 0|idle
+2017-01-11 20:57:07.100|2017-01-11 20:57:07.100| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b817142|00000000|000|000| 1| 629| 0| 0| 0|idle
+2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686246|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836867ee|00000000|000|000| 1| 908| 0| 0| 0|idle
+2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53be5f0|00000000|000|000| 1| 864| 0| 0| 0|idle
+2017-01-11 20:57:07.101|2017-01-11 20:57:07.101| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8872f85|00000000|000|000| 1| 302| 0| 0| 0|idle
+2017-01-11 20:57:07.102|2017-01-11 20:57:07.102| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39843efd|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:07.102|2017-01-11 20:57:07.102| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df52fdb|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:07.111|2017-01-11 20:57:07.111| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f503715c|00000000|000|000| 1| 1033| 0| 0| 0|idle
+2017-01-11 20:57:07.111|2017-01-11 20:57:07.111| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817383|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:07.142|2017-01-11 20:57:07.142| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6142|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:07.173|2017-01-11 20:57:07.173| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a1aae|00000000|000|000| 1| 998| 0| 0| 0|idle
+2017-01-11 20:57:07.219|2017-01-11 20:57:07.219| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df52fdb|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:07.219|2017-01-11 20:57:07.219| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df53583|00000000|000|000| 1| 189| 0| 0| 0|idle
+2017-01-11 20:57:07.219|2017-01-11 20:57:07.219| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686b46|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:07.234|2017-01-11 20:57:07.234| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6142|00000000|000|000| 1| 1067| 0| 0| 0|idle
+2017-01-11 20:57:07.234|2017-01-11 20:57:07.234| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53be91c|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:07.277|2017-01-11 20:57:07.277| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80daf97|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:07.345|2017-01-11 20:57:07.345| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628ee92|00000000|000|000| 1| 531| 0| 0| 0|idle
+2017-01-11 20:57:07.416|2017-01-11 20:57:07.416| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AP| 0| 0| 0|9b7a5871|00000000|000|000| 1| 528| 0| 0| 0|idle
+2017-01-11 20:57:07.445|2017-01-11 20:57:07.445| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d8698|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:07.519|2017-01-11 20:57:07.519| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| A| 0| 0| 0|63626dd7|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:07.538|2017-01-11 20:57:07.538| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49208| AP| 0| 0| 0|8ef414c5|00000000|000|000| 1| 526| 0| 0| 0|idle
+2017-01-11 20:57:07.551|2017-01-11 20:57:07.551| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| AP| 0| 0| 0|c0313235|00000000|000|000| 1| 395| 0| 0| 0|idle
+2017-01-11 20:57:07.551|2017-01-11 20:57:07.551| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| A| 0| 0| 0|d1fe03dc|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:07.639|2017-01-11 20:57:07.639| 0.000| 0.000| 6| 192.168.138.158|49208| 95.163.121.204| 80| A| 0| 0| 0|be149917|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:09.043|2017-01-11 20:57:09.043| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53be91c|00000000|000|000| 1| 836| 0| 0| 0|idle
+2017-01-11 20:57:09.043|2017-01-11 20:57:09.043| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6539|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:09.043|2017-01-11 20:57:09.043| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686b46|00000000|000|000| 1| 608| 0| 0| 0|idle
+2017-01-11 20:57:09.080|2017-01-11 20:57:09.080| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6539|00000000|000|000| 1| 317| 0| 0| 0|idle
+2017-01-11 20:57:09.081|2017-01-11 20:57:09.081| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bec2c|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:09.083|2017-01-11 20:57:09.083| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df5360c|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:09.085|2017-01-11 20:57:09.085| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df5360c|00000000|000|000| 1| 671| 0| 0| 0|idle
+2017-01-11 20:57:09.085|2017-01-11 20:57:09.085| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686d72|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:09.525|2017-01-11 20:57:09.525| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49207| AP| 0| 0| 0|d1fe03dc|00000000|000|000| 1| 623| 0| 0| 0|idle
+2017-01-11 20:57:09.628|2017-01-11 20:57:09.628| 0.000| 0.000| 6| 192.168.138.158|49207| 95.163.121.204| 80| A| 0| 0| 0|c0313398|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:09.697|2017-01-11 20:57:09.697| 0.000| 0.000| 6| 192.168.66.1|50186| 192.168.66.121| 8080| A| 0| 0| 0|e6b28585|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:09.697|2017-01-11 20:57:09.697| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50186| A| 0| 0| 0|79c219f0|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:12.491|2017-01-11 20:57:12.491| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| AP| 0| 0| 0|32c0b718|00000000|000|000| 1| 769| 0| 0| 0|idle
+2017-01-11 20:57:12.492|2017-01-11 20:57:12.492| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| A| 0| 0| 0|f31127d7|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:12.639|2017-01-11 20:57:12.639| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686d72|00000000|000|000| 1| 628| 0| 0| 0|idle
+2017-01-11 20:57:12.639|2017-01-11 20:57:12.639| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df53877|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:12.640|2017-01-11 20:57:12.640| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bec2c|00000000|000|000| 1| 678| 0| 0| 0|idle
+2017-01-11 20:57:12.644|2017-01-11 20:57:12.644| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df53877|00000000|000|000| 1| 501| 0| 0| 0|idle
+2017-01-11 20:57:12.644|2017-01-11 20:57:12.644| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83686fb2|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:12.650|2017-01-11 20:57:12.650| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6642|00000000|000|000| 1| 544| 0| 0| 0|idle
+2017-01-11 20:57:12.650|2017-01-11 20:57:12.650| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bee9e|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bee9e|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf446|00000000|000|000| 1| 136| 0| 0| 0|idle
+2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c682e|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83686fb2|00000000|000|000| 1| 504| 0| 0| 0|idle
+2017-01-11 20:57:13.638|2017-01-11 20:57:13.638| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b817383|00000000|000|000| 1| 580| 0| 0| 0|idle
+2017-01-11 20:57:13.639|2017-01-11 20:57:13.639| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|39843efd|00000000|000|000| 1| 629| 0| 0| 0|idle
+2017-01-11 20:57:13.640|2017-01-11 20:57:13.640| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f5037531|00000000|000|000| 1| 302| 0| 0| 0|idle
+2017-01-11 20:57:13.640|2017-01-11 20:57:13.640| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817593|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.642|2017-01-11 20:57:13.642| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c682e|00000000|000|000| 1| 678| 0| 0| 0|idle
+2017-01-11 20:57:13.642|2017-01-11 20:57:13.642| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bf49a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.646|2017-01-11 20:57:13.646| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a887307f|00000000|000|000| 1| 1032| 0| 0| 0|idle
+2017-01-11 20:57:13.647|2017-01-11 20:57:13.647| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|3984413e|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.657|2017-01-11 20:57:13.657| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df53a38|00000000|000|000| 1| 2948| 0| 0| 0|idle
+2017-01-11 20:57:13.657|2017-01-11 20:57:13.657| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.657|2017-01-11 20:57:13.657| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df54588|00000000|000|000| 1| 150| 0| 0| 0|idle
+2017-01-11 20:57:13.658|2017-01-11 20:57:13.658| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:13.686|2017-01-11 20:57:13.686| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:14.652|2017-01-11 20:57:14.652| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687176|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:14.652|2017-01-11 20:57:14.652| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|8368771e|00000000|000|000| 1| 908| 0| 0| 0|idle
+2017-01-11 20:57:14.652|2017-01-11 20:57:14.652| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df545ea|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:14.653|2017-01-11 20:57:14.653| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|3984413e|00000000|000|000| 1| 864| 0| 0| 0|idle
+2017-01-11 20:57:14.693|2017-01-11 20:57:14.693| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873453|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:14.719|2017-01-11 20:57:14.719| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df545ea|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:14.719|2017-01-11 20:57:14.719| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df54b92|00000000|000|000| 1| 187| 0| 0| 0|idle
+2017-01-11 20:57:14.719|2017-01-11 20:57:14.719| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687a76|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:14.743|2017-01-11 20:57:14.743| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8873453|00000000|000|000| 1| 1067| 0| 0| 0|idle
+2017-01-11 20:57:14.743|2017-01-11 20:57:14.743| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|3984446a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:15.201|2017-01-11 20:57:15.201| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f31127d7|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f3112d22|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b9f1|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| A| 0| 0| 0|f311326d|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f3113821|00000000|000|000| 1| 1290| 0| 0| 0|idle
+2017-01-11 20:57:15.215|2017-01-11 20:57:15.215| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49209| AP| 0| 0| 0|f3113d03|00000000|000|000| 1| 210| 0| 0| 0|idle
+2017-01-11 20:57:15.216|2017-01-11 20:57:15.216| 0.000| 0.000| 6| 192.168.138.158|49209| 95.163.121.204| 80| A| 0| 0| 0|32c0b9f1|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:15.228|2017-01-11 20:57:15.228| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| AP| 0| 0| 0|a80daf97|00000000|000|000| 1| 573| 0| 0| 0|idle
+2017-01-11 20:57:15.228|2017-01-11 20:57:15.228| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| A| 0| 0| 0|b93a1e6c|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:17.326|2017-01-11 20:57:17.326| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49206| AP| 0| 0| 0|b93a1e6c|00000000|000|000| 1| 210| 0| 0| 0|idle
+2017-01-11 20:57:17.334|2017-01-11 20:57:17.334| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| AP| 0| 0| 0|ca8d8698|00000000|000|000| 1| 480| 0| 0| 0|idle
+2017-01-11 20:57:17.335|2017-01-11 20:57:17.335| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628f07d|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:17.335|2017-01-11 20:57:17.335| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| AP| 0| 0| 0|63626dd7|00000000|000|000| 1| 483| 0| 0| 0|idle
+2017-01-11 20:57:17.335|2017-01-11 20:57:17.335| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| A| 0| 0| 0|9b7a5a59|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:17.429|2017-01-11 20:57:17.429| 0.000| 0.000| 6| 192.168.138.158|49206| 95.163.121.204| 80| A| 0| 0| 0|a80db1ac|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:18.962|2017-01-11 20:57:18.962| 0.000| 0.000| 6| 62.75.195.236| 80| 192.168.138.158|49186| AP| 0| 0| 0|73734c1c|00000000|000|000| 1| 41| 0| 0| 0|idle
+2017-01-11 20:57:18.963|2017-01-11 20:57:18.963| 0.000| 0.000| 6| 192.168.138.158|49186| 62.75.195.236| 80| A| 0| 0| 0|516c4a0c|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:19.150|2017-01-11 20:57:19.150| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628f07d|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:19.172|2017-01-11 20:57:19.172| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| A| 0| 0| 0|0628f5c8|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:19.172|2017-01-11 20:57:19.172| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|0628fb7c|00000000|000|000| 1| 1326| 0| 0| 0|idle
+2017-01-11 20:57:19.172|2017-01-11 20:57:19.172| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d8850|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:19.346|2017-01-11 20:57:19.346| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|06290082|00000000|000|000| 1| 1395| 0| 0| 0|idle
+2017-01-11 20:57:19.360|2017-01-11 20:57:19.360| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49205| AP| 0| 0| 0|062905cd|00000000|000|000| 1| 359| 0| 0| 0|idle
+2017-01-11 20:57:19.360|2017-01-11 20:57:19.360| 0.000| 0.000| 6| 192.168.138.158|49205| 95.163.121.204| 80| A| 0| 0| 0|ca8d8850|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:19.407|2017-01-11 20:57:19.407| 0.000| 0.000| 6| 95.163.121.204| 80| 192.168.138.158|49210| AP| 0| 0| 0|9b7a5a59|00000000|000|000| 1| 1017| 0| 0| 0|idle
+2017-01-11 20:57:19.512|2017-01-11 20:57:19.512| 0.000| 0.000| 6| 192.168.138.158|49210| 95.163.121.204| 80| A| 0| 0| 0|63626f92|00000000|000|000| 1| 40| 0| 0| 0|idle
+2017-01-11 20:57:19.984|2017-01-11 20:57:19.984| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|3984446a|00000000|000|000| 1| 580| 0| 0| 0|idle
+2017-01-11 20:57:19.984|2017-01-11 20:57:19.984| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a887384a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:19.985|2017-01-11 20:57:19.985| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|83687a76|00000000|000|000| 1| 629| 0| 0| 0|idle
+2017-01-11 20:57:19.985|2017-01-11 20:57:19.985| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a887384a|00000000|000|000| 1| 302| 0| 0| 0|idle
+2017-01-11 20:57:19.985|2017-01-11 20:57:19.985| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|3984467a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:19.993|2017-01-11 20:57:19.993| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df54c19|00000000|000|000| 1| 1032| 0| 0| 0|idle
+2017-01-11 20:57:19.993|2017-01-11 20:57:19.993| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687cb7|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:22.917|2017-01-11 20:57:22.917| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83687cb7|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:22.917|2017-01-11 20:57:22.917| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|8368825f|00000000|000|000| 1| 908| 0| 0| 0|idle
+2017-01-11 20:57:22.917|2017-01-11 20:57:22.917| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df54fed|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:22.918|2017-01-11 20:57:22.918| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|3984467a|00000000|000|000| 1| 864| 0| 0| 0|idle
+2017-01-11 20:57:22.918|2017-01-11 20:57:22.918| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf49a|00000000|000|000| 1| 628| 0| 0| 0|idle
+2017-01-11 20:57:22.918|2017-01-11 20:57:22.918| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| AP| 0| 0| 0|7b817593|00000000|000|000| 1| 678| 0| 0| 0|idle
+2017-01-11 20:57:22.924|2017-01-11 20:57:22.924| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50187| AP| 0| 0| 0|f503762b|00000000|000|000| 1| 544| 0| 0| 0|idle
+2017-01-11 20:57:22.924|2017-01-11 20:57:22.924| 0.000| 0.000| 6| 192.168.66.1|50187| 192.168.66.121| 8080| A| 0| 0| 0|7b817805|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:22.926|2017-01-11 20:57:22.926| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6aa0|00000000|000|000| 1| 501| 0| 0| 0|idle
+2017-01-11 20:57:22.926|2017-01-11 20:57:22.926| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bf6da|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:22.957|2017-01-11 20:57:22.957| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873944|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:22.983|2017-01-11 20:57:22.983| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df54fed|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:22.983|2017-01-11 20:57:22.983| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df55595|00000000|000|000| 1| 186| 0| 0| 0|idle
+2017-01-11 20:57:22.983|2017-01-11 20:57:22.983| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|836885b7|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:23.002|2017-01-11 20:57:23.002| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a8873944|00000000|000|000| 1| 1067| 0| 0| 0|idle
+2017-01-11 20:57:23.003|2017-01-11 20:57:23.003| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|398449a6|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.618|2017-01-11 20:57:25.618| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| AP| 0| 0| 0|398449a6|00000000|000|000| 1| 504| 0| 0| 0|idle
+2017-01-11 20:57:25.618|2017-01-11 20:57:25.618| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873d3b|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.619|2017-01-11 20:57:25.619| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836885b7|00000000|000|000| 1| 836| 0| 0| 0|idle
+2017-01-11 20:57:25.619|2017-01-11 20:57:25.619| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf6da|00000000|000|000| 1| 608| 0| 0| 0|idle
+2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| A| 0| 0| 0|a8873d3b|00000000|000|000| 1| 2948| 0| 0| 0|idle
+2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50184| AP| 0| 0| 0|a887488b|00000000|000|000| 1| 151| 0| 0| 0|idle
+2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39844b6a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39844b6a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.633|2017-01-11 20:57:25.633| 0.000| 0.000| 6| 192.168.66.1|50184| 192.168.66.121| 8080| A| 0| 0| 0|39844b6a|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.658|2017-01-11 20:57:25.658| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df5561b|00000000|000|000| 1| 317| 0| 0| 0|idle
+2017-01-11 20:57:25.658|2017-01-11 20:57:25.658| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|836888c7|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.658|2017-01-11 20:57:25.658| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6c61|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:25.659|2017-01-11 20:57:25.659| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6c61|00000000|000|000| 1| 671| 0| 0| 0|idle
+2017-01-11 20:57:25.660|2017-01-11 20:57:25.660| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bf906|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:26.665|2017-01-11 20:57:26.665| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bf906|00000000|000|000| 1| 580| 0| 0| 0|idle
+2017-01-11 20:57:26.665|2017-01-11 20:57:26.665| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6ecc|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:26.665|2017-01-11 20:57:26.665| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836888c7|00000000|000|000| 1| 629| 0| 0| 0|idle
+2017-01-11 20:57:26.666|2017-01-11 20:57:26.666| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6ecc|00000000|000|000| 1| 302| 0| 0| 0|idle
+2017-01-11 20:57:26.666|2017-01-11 20:57:26.666| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bfb16|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:26.672|2017-01-11 20:57:26.672| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df55724|00000000|000|000| 1| 1033| 0| 0| 0|idle
+2017-01-11 20:57:26.672|2017-01-11 20:57:26.672| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83688b08|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83688b08|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| AP| 0| 0| 0|836890b0|00000000|000|000| 1| 908| 0| 0| 0|idle
+2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df55af9|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:29.819|2017-01-11 20:57:29.819| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53bfb16|00000000|000|000| 1| 864| 0| 0| 0|idle
+2017-01-11 20:57:29.859|2017-01-11 20:57:29.859| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c6fc6|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:29.880|2017-01-11 20:57:29.880| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| A| 0| 0| 0|8df55af9|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:29.880|2017-01-11 20:57:29.880| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50183| AP| 0| 0| 0|8df560a1|00000000|000|000| 1| 187| 0| 0| 0|idle
+2017-01-11 20:57:29.880|2017-01-11 20:57:29.880| 0.000| 0.000| 6| 192.168.66.1|50183| 192.168.66.121| 8080| A| 0| 0| 0|83689408|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:29.904|2017-01-11 20:57:29.904| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c6fc6|00000000|000|000| 1| 1067| 0| 0| 0|idle
+2017-01-11 20:57:29.904|2017-01-11 20:57:29.904| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bfe42|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:30.749|2017-01-11 20:57:30.749| 0.000| 0.000| 17| 192.168.66.1| 5353| 224.0.0.251| 5353| 0| 0| 0| 0|00000000|00000000|000|000| 1| 68| 0| 0| 0|idle
+2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53bfe42|00000000|000|000| 1| 1500| 0| 0| 0|idle
+2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c73bd|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53c03ea|00000000|000|000| 1| 136| 0| 0| 0|idle
+2017-01-11 20:57:30.871|2017-01-11 20:57:30.871| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| A| 0| 0| 0|185c73bd|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:30.874|2017-01-11 20:57:30.874| 0.000| 0.000| 6| 192.168.66.121| 8080| 192.168.66.1|50182| AP| 0| 0| 0|185c73bd|00000000|000|000| 1| 678| 0| 0| 0|idle
+2017-01-11 20:57:30.875|2017-01-11 20:57:30.875| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| A| 0| 0| 0|e53c043e|00000000|000|000| 1| 52| 0| 0| 0|idle
+2017-01-11 20:57:31.755|2017-01-11 20:57:31.755| 0.000| 0.000| 17| 192.168.66.1| 5353| 224.0.0.251| 5353| 0| 0| 0| 0|00000000|00000000|000|000| 1| 68| 0| 0| 0|idle
+2017-01-11 20:57:32.919|2017-01-11 20:57:32.919| 0.000| 0.000| 6| 192.168.66.1|50182| 192.168.66.121| 8080| AP| 0| 0| 0|e53c043
<TRUNCATED>
[48/50] [abbrv] metron git commit: METRON-941 native PaloAlto parser
corrupts message when having a comma in the payload (ctramnitz via
justinleet) closes apache/metron#579
Posted by rm...@apache.org.
METRON-941 native PaloAlto parser corrupts message when having a comma in the payload (ctramnitz via justinleet) closes apache/metron#579
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5f08ba0b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5f08ba0b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5f08ba0b
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 5f08ba0b1dbe6ba19e8525055f639ecdb85291fc
Parents: fa5cff2
Author: ctramnitz <ct...@users.noreply.github.com>
Authored: Fri Feb 16 13:05:06 2018 -0500
Committer: leet <le...@apache.org>
Committed: Fri Feb 16 13:05:06 2018 -0500
----------------------------------------------------------------------
Upgrading.md | 18 +
.../paloalto/BasicPaloAltoFirewallParser.java | 333 +++++++++----
.../BasicPaloAltoFirewallParserTest.java | 493 ++++++++++++++++++-
.../logData/PaloAltoFirewallParserTest.txt | 2 -
4 files changed, 718 insertions(+), 128 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/Upgrading.md
----------------------------------------------------------------------
diff --git a/Upgrading.md b/Upgrading.md
index 047b68e..19da992 100644
--- a/Upgrading.md
+++ b/Upgrading.md
@@ -19,6 +19,24 @@ limitations under the License.
This document constitutes a per-version listing of changes of
configuration which are non-backwards compatible.
+## 0.4.2 to 0.4.3
+
+### [METRON-941: native PaloAlto parser corrupts message when having a comma in the payload](https://issues.apache.org/jira/browse/METRON-941)
+While modifying the PaloAlto log parser to support logs from newer
+PAN-OS version and to not break when a message payload contains a
+comma, some field names were changed to extend the coverage, fix some
+duplicate names and change some field names to the Metron standard
+message format.
+
+Installations making use of this parser should check, if the resulting
+messages still meet their expectations and adjust downstream configurations
+(i.e. ElasticSearch template) accordingly.
+
+*Note:* Previously, the samples for the test contained a full syslog line
+(including syslog header). This did - and will continue to - create a
+broken "domain" field in the parsed message. It is recommended to only feed
+the syslog message part to the parser for now.
+
## 0.4.1 to 0.4.2
### [METRON-1277: STELLAR Add Match functionality to language](https://issues.apache.org/jira/browse/METRON-1277)
http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
index 46155b3..9051f09 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParser.java
@@ -18,6 +18,8 @@
package org.apache.metron.parsers.paloalto;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
import org.apache.metron.parsers.BasicParser;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
@@ -28,68 +30,113 @@ import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.regex.Pattern;
public class BasicPaloAltoFirewallParser extends BasicParser {
+ private static boolean empty_attribute( final String s ) {
+ return s == null || s.trim().isEmpty() || s.equals("\"\"");
+ }
+
+ private static String unquoted_attribute( String s ) {
+ s = s.trim();
+ if ( s.startsWith( "\"" ) && s.endsWith( "\"" ) )
+ return s.substring( 1, s.length( ) - 1 );
+ return s;
+ }
+
private static final Logger _LOG = LoggerFactory.getLogger
(BasicPaloAltoFirewallParser.class);
private static final long serialVersionUID = 3147090149725343999L;
public static final String PaloAltoDomain = "palo_alto_domain";
public static final String ReceiveTime = "receive_time";
- public static final String SerialNum = "serial_num";
+ public static final String SerialNum = "serial";
public static final String Type = "type";
- public static final String ThreatContentType = "threat_content_type";
+ public static final String ThreatContentType = "subtype";
public static final String ConfigVersion = "config_version";
- public static final String GenerateTime = "generate_time";
- public static final String SourceAddress = "source_address";
- public static final String DestinationAddress = "destination_address";
- public static final String NATSourceIP = "nat_source_ip";
- public static final String NATDestinationIP = "nat_destination_ip";
+ public static final String GenerateTime = "time_generated";
+ public static final String SourceAddress = "ip_src_addr"; // Palo Alto name: "src"
+ public static final String DestinationAddress = "ip_dst_addr"; // Palo Alto name: "dst"
+ public static final String NATSourceIP = "natsrc";
+ public static final String NATDestinationIP = "natdst";
public static final String Rule = "rule";
- public static final String SourceUser = "source_user";
- public static final String DestinationUser = "destination_user";
- public static final String Application = "application";
- public static final String VirtualSystem = "virtual_system";
- public static final String SourceZone = "source_zone";
- public static final String DestinationZone = "destination_zone";
- public static final String InboundInterface = "inbound_interface";
- public static final String OutboundInterface = "outbound_interface";
+ public static final String SourceUser = "srcuser";
+ public static final String DestinationUser = "dstuser";
+ public static final String Application = "app";
+ public static final String VirtualSystem = "vsys";
+ public static final String SourceZone = "from";
+ public static final String DestinationZone = "to";
+ public static final String InboundInterface = "inbound_if";
+ public static final String OutboundInterface = "outbound_if";
public static final String LogAction = "log_action";
- public static final String TimeLogged = "time_logged";
- public static final String SessionID = "session_id";
- public static final String RepeatCount = "repeat_count";
- public static final String SourcePort = "source_port";
- public static final String DestinationPort = "destination_port";
- public static final String NATSourcePort = "nats_source_port";
- public static final String NATDestinationPort = "nats_destination_port";
+ public static final String TimeLogged = "start";
+ public static final String SessionID = "sessionid";
+ public static final String RepeatCount = "repeatcnt";
+ public static final String SourcePort = "ip_src_port"; // Palo Alto name: "sport"
+ public static final String DestinationPort = "ip_dst_port"; // Palo Alto name: "dport"
+ public static final String NATSourcePort = "natsport";
+ public static final String NATDestinationPort = "natdport";
public static final String Flags = "flags";
- public static final String IPProtocol = "ip_protocol";
+ public static final String IPProtocol = "protocol"; // Palo Alto name: "proto"
public static final String Action = "action";
+ public static final String Seqno = "seqno";
+ public static final String ActionFlags = "actionflags";
+ public static final String Category = "category";
+ public static final String DGH1 = "dg_hier_level_1";
+ public static final String DGH2 = "dg_hier_level_2";
+ public static final String DGH3 = "dg_hier_level_3";
+ public static final String DGH4 = "dg_hier_level_4";
+ public static final String VSYSName = "vsys_name";
+ public static final String DeviceName = "device_name";
+ public static final String ActionSource = "action_source";
+ public static final String ParserVersion = "parser_version";
+ public static final String Tokens = "tokens_seen";
+
+ public static final String SourceVmUuid = "source_vm_uuid";
+ public static final String DestinationVmUuid = "destination_vm_uuid";
+ public static final String TunnelId = "tunnel_id";
+ public static final String MonitorTag = "monitor_tag";
+ public static final String ParentSessionId = "parent_session_id";
+ public static final String ParentSessionStartTime = "parent_session_start_time";
+ public static final String TunnelType = "tunnel_type";
//Threat
public static final String URL = "url";
public static final String HOST = "host";
- public static final String ThreatContentName = "threat_content_name";
- public static final String Category = "category";
+ public static final String ThreatID = "threatid";
+ public static final String Severity = "severity";
public static final String Direction = "direction";
- public static final String Seqno = "seqno";
- public static final String ActionFlags = "action_flags";
- public static final String SourceCountry = "source_country";
- public static final String DestinationCountry = "destination_country";
- public static final String Cpadding = "cpadding";
- public static final String ContentType = "content_type";
+ public static final String SourceLocation = "srcloc";
+ public static final String DestinationLocation = "dstloc";
+ public static final String ContentType = "contenttype";
+ public static final String PCAPID = "pcap_id";
+ public static final String WFFileDigest = "filedigest";
+ public static final String WFCloud = "cloud";
+ public static final String UserAgent= "user_agent";
+ public static final String WFFileType = "filetype";
+ public static final String XForwardedFor = "xff";
+ public static final String Referer = "referer";
+ public static final String WFSender = "sender";
+ public static final String WFSubject = "subject";
+ public static final String WFRecipient = "recipient";
+ public static final String WFReportID = "reportid";
+ public static final String URLIndex = "url_idx";
+ public static final String HTTPMethod = "http_method";
+ public static final String ThreatCategory = "threat_category";
+ public static final String ContentVersion = "content_version";
+
//Traffic
- public static final String Bytes = "content_type";
- public static final String BytesSent = "content_type";
- public static final String BytesReceived = "content_type";
- public static final String Packets = "content_type";
- public static final String StartTime = "content_type";
- public static final String ElapsedTimeInSec = "content_type";
- public static final String Padding = "content_type";
+ public static final String Bytes = "bytes";
+ public static final String BytesSent = "bytes_sent";
+ public static final String BytesReceived = "bytes_received";
+ public static final String Packets = "packets";
+ public static final String StartTime = "start";
+ public static final String ElapsedTimeInSec = "elapsed";
public static final String PktsSent = "pkts_sent";
public static final String PktsReceived = "pkts_received";
+ public static final String EndReason = "session_end_reason";
@Override
public void configure(Map<String, Object> parserConfig) {
@@ -117,12 +164,6 @@ public class BasicPaloAltoFirewallParser extends BasicParser {
parseMessage(toParse, outputMessage);
long timestamp = System.currentTimeMillis();
outputMessage.put("timestamp", System.currentTimeMillis());
- outputMessage.put("ip_src_addr", outputMessage.remove("source_address"));
- outputMessage.put("ip_src_port", outputMessage.remove("source_port"));
- outputMessage.put("ip_dst_addr", outputMessage.remove("destination_address"));
- outputMessage.put("ip_dst_port", outputMessage.remove("destination_port"));
- outputMessage.put("protocol", outputMessage.remove("ip_protocol"));
-
outputMessage.put("original_string", toParse);
messages.add(outputMessage);
return messages;
@@ -136,77 +177,157 @@ public class BasicPaloAltoFirewallParser extends BasicParser {
@SuppressWarnings("unchecked")
private void parseMessage(String message, JSONObject outputMessage) {
- String[] tokens = message.split(",");
+ String[] tokens = Iterables.toArray(Splitter.on(Pattern.compile(",(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$)")).split(message), String.class);
+ int parser_version = 0;
String type = tokens[3].trim();
//populate common objects
- outputMessage.put(PaloAltoDomain, tokens[0].trim());
- outputMessage.put(ReceiveTime, tokens[1].trim());
- outputMessage.put(SerialNum, tokens[2].trim());
+ if( !empty_attribute( tokens[0] ) ) outputMessage.put(PaloAltoDomain, tokens[0].trim());
+ if( !empty_attribute( tokens[1] ) ) outputMessage.put(ReceiveTime, tokens[1].trim());
+ if( !empty_attribute( tokens[2] ) ) outputMessage.put(SerialNum, tokens[2].trim());
outputMessage.put(Type, type);
- outputMessage.put(ThreatContentType, tokens[4].trim());
- outputMessage.put(ConfigVersion, tokens[5].trim());
- outputMessage.put(GenerateTime, tokens[6].trim());
- outputMessage.put(SourceAddress, tokens[7].trim());
- outputMessage.put(DestinationAddress, tokens[8].trim());
- outputMessage.put(NATSourceIP, tokens[9].trim());
- outputMessage.put(NATDestinationIP, tokens[10].trim());
- outputMessage.put(Rule, tokens[11].trim());
- outputMessage.put(SourceUser, tokens[12].trim());
- outputMessage.put(DestinationUser, tokens[13].trim());
- outputMessage.put(Application, tokens[14].trim());
- outputMessage.put(VirtualSystem, tokens[15].trim());
- outputMessage.put(SourceZone, tokens[16].trim());
- outputMessage.put(DestinationZone, tokens[17].trim());
- outputMessage.put(InboundInterface, tokens[18].trim());
- outputMessage.put(OutboundInterface, tokens[19].trim());
- outputMessage.put(LogAction, tokens[20].trim());
- outputMessage.put(TimeLogged, tokens[21].trim());
- outputMessage.put(SessionID, tokens[22].trim());
- outputMessage.put(RepeatCount, tokens[23].trim());
- outputMessage.put(SourcePort, tokens[24].trim());
- outputMessage.put(DestinationPort, tokens[25].trim());
- outputMessage.put(NATSourcePort, tokens[26].trim());
- outputMessage.put(NATDestinationPort, tokens[27].trim());
- outputMessage.put(Flags, tokens[28].trim());
- outputMessage.put(IPProtocol, tokens[29].trim());
- outputMessage.put(Action, tokens[30].trim());
+ if( !empty_attribute( tokens[4] ) ) outputMessage.put(ThreatContentType, unquoted_attribute(tokens[4]));
+ if( !empty_attribute( tokens[5] ) ) outputMessage.put(ConfigVersion, tokens[5].trim());
+ if( !empty_attribute( tokens[6] ) ) outputMessage.put(GenerateTime, tokens[6].trim());
+ if( !empty_attribute( tokens[7] ) ) outputMessage.put(SourceAddress, tokens[7].trim());
+ if( !empty_attribute( tokens[8] ) ) outputMessage.put(DestinationAddress, tokens[8].trim());
+ if( !empty_attribute( tokens[9] ) ) outputMessage.put(NATSourceIP, tokens[9].trim());
+ if( !empty_attribute( tokens[10] ) ) outputMessage.put(NATDestinationIP, tokens[10].trim());
+ if( !empty_attribute( tokens[11] ) ) outputMessage.put(Rule, unquoted_attribute(tokens[11]));
+ if( !empty_attribute( tokens[12] ) ) outputMessage.put(SourceUser, unquoted_attribute(tokens[12]));
+ if( !empty_attribute( tokens[13] ) ) outputMessage.put(DestinationUser, unquoted_attribute(tokens[13]));
+ if( !empty_attribute( tokens[14] ) ) outputMessage.put(Application, unquoted_attribute(tokens[14]));
+ if( !empty_attribute( tokens[15] ) ) outputMessage.put(VirtualSystem, unquoted_attribute(tokens[15]));
+ if( !empty_attribute( tokens[16] ) ) outputMessage.put(SourceZone, unquoted_attribute(tokens[16]));
+ if( !empty_attribute( tokens[17] ) ) outputMessage.put(DestinationZone, unquoted_attribute(tokens[17]));
+ if( !empty_attribute( tokens[18] ) ) outputMessage.put(InboundInterface, unquoted_attribute(tokens[18]));
+ if( !empty_attribute( tokens[19] ) ) outputMessage.put(OutboundInterface, unquoted_attribute(tokens[19]));
+ if( !empty_attribute( tokens[20] ) ) outputMessage.put(LogAction, unquoted_attribute(tokens[20]));
+ if( !empty_attribute( tokens[21] ) ) outputMessage.put(TimeLogged, tokens[21].trim());
+ if( !empty_attribute( tokens[22] ) ) outputMessage.put(SessionID, tokens[22].trim());
+ if( !empty_attribute( tokens[23] ) ) outputMessage.put(RepeatCount, tokens[23].trim());
+ if( !empty_attribute( tokens[24] ) ) outputMessage.put(SourcePort, tokens[24].trim());
+ if( !empty_attribute( tokens[25] ) ) outputMessage.put(DestinationPort, tokens[25].trim());
+ if( !empty_attribute( tokens[26] ) ) outputMessage.put(NATSourcePort, tokens[26].trim());
+ if( !empty_attribute( tokens[27] ) ) outputMessage.put(NATDestinationPort, tokens[27].trim());
+ if( !empty_attribute( tokens[28] ) ) outputMessage.put(Flags, tokens[28].trim());
+ if( !empty_attribute( tokens[29] ) ) outputMessage.put(IPProtocol, unquoted_attribute(tokens[29]));
+ if( !empty_attribute( tokens[30] ) ) outputMessage.put(Action, unquoted_attribute(tokens[30]));
if ("THREAT".equals(type.toUpperCase())) {
- outputMessage.put(URL, tokens[31].trim());
- try {
- URL url = new URL(tokens[31].trim());
- outputMessage.put(HOST, url.getHost());
- } catch (MalformedURLException e) {
+ int p1_offset = 0;
+ if (tokens.length == 45) parser_version = 60;
+ else if (tokens.length == 53) parser_version = 61;
+ else if (tokens.length == 61) {
+ parser_version = 70;
+ p1_offset = 1;
+ }
+ else if (tokens.length == 72) {
+ parser_version = 80;
+ p1_offset =1;
+ }
+ outputMessage.put(ParserVersion, parser_version);
+ if( !empty_attribute( tokens[31] ) ) {
+ outputMessage.put(URL, unquoted_attribute(tokens[31]));
+ try {
+ URL url = new URL(unquoted_attribute(tokens[31]));
+ outputMessage.put(HOST, url.getHost());
+ } catch (MalformedURLException e) {
+ }
+ }
+ if( !empty_attribute( tokens[32] ) ) outputMessage.put(ThreatID, tokens[32].trim());
+ if( !empty_attribute( tokens[33] ) ) outputMessage.put(Category, unquoted_attribute(tokens[33]));
+ if( !empty_attribute( tokens[34] ) ) outputMessage.put(Severity, unquoted_attribute(tokens[34]));
+ if( !empty_attribute( tokens[35] ) ) outputMessage.put(Direction, unquoted_attribute(tokens[35]));
+ if( !empty_attribute( tokens[36] ) ) outputMessage.put(Seqno, tokens[36].trim());
+ if( !empty_attribute( tokens[37] ) ) outputMessage.put(ActionFlags, unquoted_attribute(tokens[37]));
+ if( !empty_attribute( tokens[38] ) ) outputMessage.put(SourceLocation, unquoted_attribute(tokens[38]));
+ if( !empty_attribute( tokens[39] ) ) outputMessage.put(DestinationLocation, unquoted_attribute(tokens[39]));
+ if( !empty_attribute( tokens[41] ) ) outputMessage.put(ContentType, unquoted_attribute(tokens[41]));
+ if( !empty_attribute( tokens[42] ) ) outputMessage.put(PCAPID, tokens[42].trim());
+ if( !empty_attribute( tokens[43] ) ) outputMessage.put(WFFileDigest, unquoted_attribute(tokens[43]));
+ if( !empty_attribute( tokens[44] ) ) outputMessage.put(WFCloud, unquoted_attribute(tokens[44]));
+ if ( parser_version >= 61) {
+ if( !empty_attribute( tokens[(45 + p1_offset)] ) ) outputMessage.put(UserAgent, unquoted_attribute(tokens[(45 + p1_offset)]));
+ if( !empty_attribute( tokens[(46 + p1_offset)] ) ) outputMessage.put(WFFileType, unquoted_attribute(tokens[(46 + p1_offset)]));
+ if( !empty_attribute( tokens[(47 + p1_offset)] ) ) outputMessage.put(XForwardedFor, unquoted_attribute(tokens[(47 + p1_offset)]));
+ if( !empty_attribute( tokens[(48 + p1_offset)] ) ) outputMessage.put(Referer, unquoted_attribute(tokens[(48 + p1_offset)]));
+ if( !empty_attribute( tokens[(49 + p1_offset)] ) ) outputMessage.put(WFSender, unquoted_attribute(tokens[(49 + p1_offset)]));
+ if( !empty_attribute( tokens[(50 + p1_offset)] ) ) outputMessage.put(WFSubject, unquoted_attribute(tokens[(50 + p1_offset)]));
+ if( !empty_attribute( tokens[(51 + p1_offset)] ) ) outputMessage.put(WFRecipient, unquoted_attribute(tokens[(51 + p1_offset)]));
+ if( !empty_attribute( tokens[(52 + p1_offset)] ) ) outputMessage.put(WFReportID, unquoted_attribute(tokens[(52 + p1_offset)]));
+ }
+ if ( parser_version >= 70) {
+ if( !empty_attribute( tokens[45] ) ) outputMessage.put(URLIndex, tokens[45].trim());
+ if( !empty_attribute( tokens[54] ) ) outputMessage.put(DGH1, tokens[54].trim());
+ if( !empty_attribute( tokens[55] ) ) outputMessage.put(DGH2, tokens[55].trim());
+ if( !empty_attribute( tokens[56] ) ) outputMessage.put(DGH3, tokens[56].trim());
+ if( !empty_attribute( tokens[57] ) ) outputMessage.put(DGH4, tokens[57].trim());
+ if( !empty_attribute( tokens[58] ) ) outputMessage.put(VSYSName, unquoted_attribute(tokens[58]));
+ if( !empty_attribute( tokens[59] ) ) outputMessage.put(DeviceName, unquoted_attribute(tokens[59]));
+ }
+ if ( parser_version >= 80) {
+ if( !empty_attribute( tokens[61] ) ) outputMessage.put(SourceVmUuid, tokens[61].trim());
+ if( !empty_attribute( tokens[62] ) ) outputMessage.put(DestinationVmUuid, tokens[62].trim());
+ if( !empty_attribute( tokens[63] ) ) outputMessage.put(HTTPMethod, tokens[63].trim());
+ if( !empty_attribute( tokens[64] ) ) outputMessage.put(TunnelId, tokens[64].trim());
+ if( !empty_attribute( tokens[65] ) ) outputMessage.put(MonitorTag, tokens[65].trim());
+ if( !empty_attribute( tokens[66] ) ) outputMessage.put(ParentSessionId, tokens[66].trim());
+ if( !empty_attribute( tokens[67] ) ) outputMessage.put(ParentSessionStartTime, tokens[67].trim());
+ if( !empty_attribute( tokens[68] ) ) outputMessage.put(TunnelType, tokens[68].trim());
+ if( !empty_attribute( tokens[69] ) ) outputMessage.put(ThreatCategory, tokens[69].trim());
+ if( !empty_attribute( tokens[70] ) ) outputMessage.put(ContentVersion, tokens[70].trim());
+ }
+ if ( parser_version == 0) {
+ outputMessage.put(Tokens, tokens.length);
+ }
+
+
+ } else if ("TRAFFIC".equals(type.toUpperCase())) {
+ if (tokens.length == 46) parser_version = 60;
+ else if (tokens.length == 47) parser_version = 61;
+ else if (tokens.length == 54) parser_version = 70;
+ else if (tokens.length == 61) parser_version = 80;
+ outputMessage.put(ParserVersion, parser_version);
+ if( !empty_attribute( tokens[31] ) ) outputMessage.put(Bytes, tokens[31].trim());
+ if( !empty_attribute( tokens[32] ) ) outputMessage.put(BytesSent, tokens[32].trim());
+ if( !empty_attribute( tokens[33] ) ) outputMessage.put(BytesReceived, tokens[33].trim());
+ if( !empty_attribute( tokens[34] ) ) outputMessage.put(Packets, tokens[34].trim());
+ if( !empty_attribute( tokens[35] ) ) outputMessage.put(StartTime, tokens[35].trim());
+ if( !empty_attribute( tokens[36] ) ) outputMessage.put(ElapsedTimeInSec, tokens[36].trim());
+ if( !empty_attribute( tokens[37] ) ) outputMessage.put(Category, unquoted_attribute(tokens[37]));
+ if( !empty_attribute( tokens[39] ) ) outputMessage.put(Seqno, tokens[39].trim());
+ if( !empty_attribute( tokens[40] ) ) outputMessage.put(ActionFlags, unquoted_attribute(tokens[40]));
+ if( !empty_attribute( tokens[41] ) ) outputMessage.put(SourceLocation, unquoted_attribute(tokens[41]));
+ if( !empty_attribute( tokens[42] ) ) outputMessage.put(DestinationLocation, unquoted_attribute(tokens[42]));
+ if( !empty_attribute( tokens[44] ) ) outputMessage.put(PktsSent, tokens[44].trim());
+ if( !empty_attribute( tokens[45] ) ) outputMessage.put(PktsReceived, tokens[45].trim());
+ if ( parser_version >= 61) {
+ if( !empty_attribute( tokens[46] ) ) outputMessage.put(EndReason, unquoted_attribute(tokens[46]));
+ }
+ if ( parser_version >= 70) {
+ if( !empty_attribute( tokens[47] ) ) outputMessage.put(DGH1, tokens[47].trim());
+ if( !empty_attribute( tokens[48] ) ) outputMessage.put(DGH2, tokens[48].trim());
+ if( !empty_attribute( tokens[49] ) ) outputMessage.put(DGH3, tokens[49].trim());
+ if( !empty_attribute( tokens[50] ) ) outputMessage.put(DGH4, tokens[50].trim());
+ if( !empty_attribute( tokens[51] ) ) outputMessage.put(VSYSName, unquoted_attribute(tokens[51]));
+ if( !empty_attribute( tokens[52] ) ) outputMessage.put(DeviceName, unquoted_attribute(tokens[52]));
+ if( !empty_attribute( tokens[53] ) ) outputMessage.put(ActionSource, unquoted_attribute(tokens[53]));
+ }
+ if ( parser_version >= 80) {
+ if( !empty_attribute( tokens[54] ) ) outputMessage.put(SourceVmUuid, tokens[54].trim());
+ if( !empty_attribute( tokens[55] ) ) outputMessage.put(DestinationVmUuid, tokens[55].trim());
+ if( !empty_attribute( tokens[56] ) ) outputMessage.put(TunnelId, tokens[56].trim());
+ if( !empty_attribute( tokens[57] ) ) outputMessage.put(MonitorTag, tokens[57].trim());
+ if( !empty_attribute( tokens[58] ) ) outputMessage.put(ParentSessionId, tokens[58].trim());
+ if( !empty_attribute( tokens[59] ) ) outputMessage.put(ParentSessionStartTime, tokens[59].trim());
+ if( !empty_attribute( tokens[60] ) ) outputMessage.put(TunnelType, tokens[60].trim());
+ }
+ if ( parser_version == 0) {
+ outputMessage.put(Tokens, tokens.length);
}
- outputMessage.put(ThreatContentName, tokens[32].trim());
- outputMessage.put(Category, tokens[33].trim());
- outputMessage.put(Direction, tokens[34].trim());
- outputMessage.put(Seqno, tokens[35].trim());
- outputMessage.put(ActionFlags, tokens[36].trim());
- outputMessage.put(SourceCountry, tokens[37].trim());
- outputMessage.put(DestinationCountry, tokens[38].trim());
- outputMessage.put(Cpadding, tokens[39].trim());
- outputMessage.put(ContentType, tokens[40].trim());
-
- } else {
- outputMessage.put(Bytes, tokens[31].trim());
- outputMessage.put(BytesSent, tokens[32].trim());
- outputMessage.put(BytesReceived, tokens[33].trim());
- outputMessage.put(Packets, tokens[34].trim());
- outputMessage.put(StartTime, tokens[35].trim());
- outputMessage.put(ElapsedTimeInSec, tokens[36].trim());
- outputMessage.put(Category, tokens[37].trim());
- outputMessage.put(Padding, tokens[38].trim());
- outputMessage.put(Seqno, tokens[39].trim());
- outputMessage.put(ActionFlags, tokens[40].trim());
- outputMessage.put(SourceCountry, tokens[41].trim());
- outputMessage.put(DestinationCountry, tokens[42].trim());
- outputMessage.put(Cpadding, tokens[43].trim());
- outputMessage.put(PktsSent, tokens[44].trim());
- outputMessage.put(PktsReceived, tokens[45].trim());
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
index cf93c92..2c90b1e 100644
--- a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
+++ b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/paloalto/BasicPaloAltoFirewallParserTest.java
@@ -17,13 +17,11 @@
*/
package org.apache.metron.parsers.paloalto;
-import java.util.Map;
-import java.util.Map.Entry;
+import static org.junit.Assert.assertEquals;
+
import org.apache.metron.parsers.AbstractParserConfigTest;
import org.json.simple.JSONObject;
-import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
-import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -31,27 +29,482 @@ public class BasicPaloAltoFirewallParserTest extends AbstractParserConfigTest {
@Before
public void setUp() throws Exception {
- inputStrings = readTestDataFromFile(
- "src/test/resources/logData/PaloAltoFirewallParserTest.txt");
parser = new BasicPaloAltoFirewallParser();
}
- @SuppressWarnings({"rawtypes"})
+ public static final String THREAT_60 = "1,2015/01/05 05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP: IIS Denial Of Service Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,,";
+
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseThreat60() throws ParseException {
+ JSONObject actual = parser.parse(THREAT_60.getBytes()).get(0);
+
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "reset-both");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.Application, "web-browsing");
+ expected.put(BasicPaloAltoFirewallParser.Category, "any");
+
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+ expected.put(BasicPaloAltoFirewallParser.Direction, "client-to-server");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "US");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x80004000");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "internal");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/2");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "216.0.10.198");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "80");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "10.0.0.115");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "54180");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "LOG-Default");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0");
+ expected.put("original_string", THREAT_60);
+ expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/1");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 60);
+ expected.put(BasicPaloAltoFirewallParser.PCAPID, "1200568889751109656");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2015/01/05 05:38:58");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "EX-Allow");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "347368099");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "0006C110285");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "12031");
+ expected.put(BasicPaloAltoFirewallParser.Severity, "high");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "10.0.0.0-10.255.255.255");
+ expected.put(BasicPaloAltoFirewallParser.SourceUser, "example\\user.name");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2015/01/05 05:38:58");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "vulnerability");
+ expected.put(BasicPaloAltoFirewallParser.ThreatID, "HTTP: IIS Denial Of Service Attempt(40019)");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2015/01/05 05:38:58");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "external");
+ expected.put(BasicPaloAltoFirewallParser.Type, "THREAT");
+ expected.put(BasicPaloAltoFirewallParser.URL, "ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String TRAFFIC_60 = "1,2015/01/05 12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,";
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseTraffic60() throws ParseException {
+ JSONObject actual = parser.parse(TRAFFIC_60.getBytes()).get(0);
+
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "allow");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.Application, "ms-ds-smb");
+ expected.put(BasicPaloAltoFirewallParser.Bytes, "2229");
+ expected.put(BasicPaloAltoFirewallParser.BytesReceived, "942");
+ expected.put(BasicPaloAltoFirewallParser.BytesSent, "1287");
+ expected.put(BasicPaloAltoFirewallParser.Category, "any");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "10.0.0.0-10.255.255.255");
+ expected.put(BasicPaloAltoFirewallParser.DestinationUser, "example\\\\user.name");
+ expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "30");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x401a");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "v_external");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/2");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "10.1.0.163");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "445");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "10.0.0.39");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "52688");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "LOG-Default");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0");
+ expected.put("original_string", TRAFFIC_60);
+ expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/1");
+ expected.put(BasicPaloAltoFirewallParser.Packets, "10");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 60);
+ expected.put(BasicPaloAltoFirewallParser.PktsSent, "6");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2015/01/05 12:51:33");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "EX-Allow");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "17754932062");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "0011C103117");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "33760927");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "10.0.0.0-10.255.255.255");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2015/01/05 12:51:01");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "end");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2015/01/05 12:51:33");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "v_internal");
+ expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String THREAT_70 = "1,2017/05/24 09:53:10,001801000001,THREAT,virus,0,2017/05/24 09:53:10,217.1.2.3,10.1.8.7,217.1.2.3,214.123.1.2,WLAN-Internet,,user,web-browsing,vsys1,Untrust,wifi_zone,ethernet1/1,vlan.1,Std-Log-Forward,2017/05/24 09:53:10,49567,1,80,51787,80,25025,0x400000,tcp,reset-both,\"abcdef310.exe\",Virus/Win32.WGeneric.lumeo(2457399),computer-and-internet-info,medium,server-to-client,329423829,0x0,DE,10.0.0.0-10.255.255.255,0,,0,,,1,,,\"\",\"\",,,,0,19,0,0,0,,PAN1,";
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseThreat70() throws ParseException {
+ JSONObject actual = parser.parse(THREAT_70.getBytes()).get(0);
+
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "reset-both");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.Application, "web-browsing");
+ expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "0");
+ expected.put(BasicPaloAltoFirewallParser.Direction, "server-to-client");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "10.0.0.0-10.255.255.255");
+ expected.put(BasicPaloAltoFirewallParser.DestinationUser, "user");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x400000");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "Untrust");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "10.1.8.7");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "51787");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "217.1.2.3");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "80");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "Std-Log-Forward");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "25025");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "214.123.1.2");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "80");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "217.1.2.3");
+ expected.put("original_string", THREAT_70);
+ expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "vlan.1");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70);
+ expected.put(BasicPaloAltoFirewallParser.PCAPID, "0");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/24 09:53:10");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "WLAN-Internet");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "329423829");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "001801000001");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "49567");
+ expected.put(BasicPaloAltoFirewallParser.Severity, "medium");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "DE");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/24 09:53:10");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "virus");
+ expected.put(BasicPaloAltoFirewallParser.ThreatID, "Virus/Win32.WGeneric.lumeo(2457399)");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/24 09:53:10");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "wifi_zone");
+ expected.put(BasicPaloAltoFirewallParser.Type, "THREAT");
+ expected.put(BasicPaloAltoFirewallParser.URL, "abcdef310.exe");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ expected.put(BasicPaloAltoFirewallParser.URLIndex, "1");
+ expected.put(BasicPaloAltoFirewallParser.WFReportID, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH1, "19");
+ expected.put(BasicPaloAltoFirewallParser.DGH2, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH3, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+ expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String TRAFFIC_70 = "1,2017/05/25 21:38:13,001606000003,TRAFFIC,drop,1,2017/05/25 21:38:13,10.2.1.8,192.168.1.10,0.0.0.0,0.0.0.0,DropLog,,,not-applicable,vsys1,intern,VPN,vlan.1,,Std-Log-Forward,2017/05/25 21:38:13,0,1,137,137,0,0,0x0,udp,deny,114,114,0,1,2017/05/25 21:38:12,0,any,0,9953744,0x0,192.168.0.0-192.168.255.255,DE,0,1,0,policy-deny,19,0,0,0,,PAN1,from-policy";
+ @SuppressWarnings("unchecked")
@Test
- public void testParse() throws ParseException {
- for (String inputString : inputStrings) {
- JSONObject parsed = parser.parse(inputString.getBytes()).get(0);
- Assert.assertNotNull(parsed);
+ public void testParseTraffic70() throws ParseException {
+ JSONObject actual = parser.parse(TRAFFIC_70.getBytes()).get(0);
- JSONParser parser = new JSONParser();
- Map json = (Map) parser.parse(parsed.toJSONString());
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "deny");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.ActionSource, "from-policy");
+ expected.put(BasicPaloAltoFirewallParser.Application, "not-applicable");
+ expected.put(BasicPaloAltoFirewallParser.Bytes, "114");
+ expected.put(BasicPaloAltoFirewallParser.BytesReceived, "0");
+ expected.put(BasicPaloAltoFirewallParser.BytesSent, "114");
+ expected.put(BasicPaloAltoFirewallParser.Category, "any");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "DE");
+ expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "0");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "intern");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "vlan.1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "192.168.1.10");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "137");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "10.2.1.8");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "137");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "Std-Log-Forward");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0");
+ expected.put("original_string", TRAFFIC_70);
+ expected.put(BasicPaloAltoFirewallParser.Packets, "1");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70);
+ expected.put(BasicPaloAltoFirewallParser.PktsReceived, "0");
+ expected.put(BasicPaloAltoFirewallParser.PktsSent, "1");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "udp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/25 21:38:13");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "DropLog");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "9953744");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "001606000003");
+ expected.put(BasicPaloAltoFirewallParser.EndReason, "policy-deny");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "0");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "192.168.0.0-192.168.255.255");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/25 21:38:12");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "drop");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/25 21:38:13");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "VPN");
+ expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ expected.put(BasicPaloAltoFirewallParser.DGH1, "19");
+ expected.put(BasicPaloAltoFirewallParser.DGH2, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH3, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+ expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String TRAFFIC_71 = "1,2017/05/31 23:59:57,0006C000005,TRAFFIC,drop,0,2017/05/31 23:59:57,185.94.1.1,201.1.4.5,0.0.0.0,0.0.0.0,DropLog,,,not-applicable,vsys1,untrust,untrust,vlan.1,,Standard-Syslog,2017/05/31 23:59:57,0,1,59836,123,0,0,0x0,udp,deny,60,60,0,1,2017/05/31 23:59:57,0,any,0,3433072193,0x0,RU,DE,0,1,0,policy-deny,16,11,0,0,,PAN1,from-policy";
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseTraffic71() throws ParseException {
+ JSONObject actual = parser.parse(TRAFFIC_71.getBytes()).get(0);
+
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "deny");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.ActionSource, "from-policy");
+ expected.put(BasicPaloAltoFirewallParser.Application, "not-applicable");
+ expected.put(BasicPaloAltoFirewallParser.Bytes, "60");
+ expected.put(BasicPaloAltoFirewallParser.BytesReceived, "0");
+ expected.put(BasicPaloAltoFirewallParser.BytesSent, "60");
+ expected.put(BasicPaloAltoFirewallParser.Category, "any");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "0");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "DE");
+ expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "0");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "untrust");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "vlan.1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "201.1.4.5");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "123");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "185.94.1.1");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "59836");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "Standard-Syslog");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "0.0.0.0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "0");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "0.0.0.0");
+ expected.put("original_string", TRAFFIC_71);
+ expected.put(BasicPaloAltoFirewallParser.Packets, "1");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70);
+ expected.put(BasicPaloAltoFirewallParser.PktsReceived, "0");
+ expected.put(BasicPaloAltoFirewallParser.PktsSent, "1");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "udp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/31 23:59:57");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "DropLog");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "3433072193");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "0006C000005");
+ expected.put(BasicPaloAltoFirewallParser.EndReason, "policy-deny");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "0");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "RU");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/31 23:59:57");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "drop");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/31 23:59:57");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "untrust");
+ expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ expected.put(BasicPaloAltoFirewallParser.DGH1, "16");
+ expected.put(BasicPaloAltoFirewallParser.DGH2, "11");
+ expected.put(BasicPaloAltoFirewallParser.DGH3, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+ expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String THREAT_71 = "1,2017/05/25 19:31:13,0006C000005,THREAT,url,0,2017/05/25 19:31:13,192.168.1.7,140.177.26.29,201.1.4.5,140.177.26.29,ms_out,,,ssl,vsys1,mgmt,untrust,vlan.199,vlan.1,Standard-Syslog,2017/05/25 19:31:13,50556,1,56059,443,14810,443,0x40b000,tcp,alert,\"settings-win.data.microsoft.com/\",(9999),computer-and-internet-info,informational,client-to-server,10030265,0x0,192.168.0.0-192.168.255.255,IE,0,,0,,,0,,,,,,,,0,16,11,0,0,,PAN1,";
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseThreat71() throws ParseException {
+ JSONObject actual = parser.parse(THREAT_71.getBytes()).get(0);
+
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "alert");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.Application, "ssl");
+ expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "0");
+ expected.put(BasicPaloAltoFirewallParser.Direction, "client-to-server");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "IE");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x40b000");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "mgmt");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "vlan.199");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "140.177.26.29");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "443");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "192.168.1.7");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "56059");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "Standard-Syslog");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "443");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "140.177.26.29");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "14810");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "201.1.4.5");
+ expected.put("original_string", THREAT_71);
+ expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "vlan.1");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 70);
+ expected.put(BasicPaloAltoFirewallParser.PCAPID, "0");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2017/05/25 19:31:13");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "ms_out");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "10030265");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "0006C000005");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "50556");
+ expected.put(BasicPaloAltoFirewallParser.Severity, "informational");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "192.168.0.0-192.168.255.255");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2017/05/25 19:31:13");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "url");
+ expected.put(BasicPaloAltoFirewallParser.ThreatID, "(9999)");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2017/05/25 19:31:13");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "untrust");
+ expected.put(BasicPaloAltoFirewallParser.Type, "THREAT");
+ expected.put(BasicPaloAltoFirewallParser.URL, "settings-win.data.microsoft.com/");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ expected.put(BasicPaloAltoFirewallParser.URLIndex, "0");
+ expected.put(BasicPaloAltoFirewallParser.WFReportID, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH1, "16");
+ expected.put(BasicPaloAltoFirewallParser.DGH2, "11");
+ expected.put(BasicPaloAltoFirewallParser.DGH3, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+ expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String THREAT_80 = "1,2018/02/01 21:29:03,001606000007,THREAT,vulnerability,1,2018/02/01 21:29:03,213.211.198.62,172.16.2.6,213.211.198.62,192.168.178.202,Outgoing,,,web-browsing,vsys1,internet,guest,ethernet1/1,ethernet1/2.2,test,2018/02/01 21:29:03,18720,1,80,53161,80,32812,0x402000,tcp,reset-server,\"www.eicar.org/download/eicar.com\",Eicar File Detected(39040),computer-and-internet-info,medium,server-to-client,27438839,0x0,Germany,172.16.0.0-172.31.255.255,0,,0,,,9,,,,,,,,0,0,0,0,0,,PAN1,,,,,0,,0,,N/A,code-execution,AppThreat-771-4450,0x0";
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseThreat80() throws ParseException {
+ JSONObject actual = parser.parse(THREAT_80.getBytes()).get(0);
+
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "reset-server");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.Application, "web-browsing");
+ expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+ expected.put(BasicPaloAltoFirewallParser.ContentVersion, "AppThreat-771-4450");
+ expected.put(BasicPaloAltoFirewallParser.Direction, "server-to-client");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "172.16.0.0-172.31.255.255");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x402000");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "internet");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "172.16.2.6");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "53161");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "213.211.198.62");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "80");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "test");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "32812");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "192.168.178.202");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "80");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "213.211.198.62");
+ expected.put("original_string", THREAT_80);
+ expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/2.2");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParentSessionId, "0");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80);
+ expected.put(BasicPaloAltoFirewallParser.PCAPID, "0");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2018/02/01 21:29:03");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "Outgoing");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "27438839");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "001606000007");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "18720");
+ expected.put(BasicPaloAltoFirewallParser.Severity, "medium");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "Germany");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2018/02/01 21:29:03");
+ expected.put(BasicPaloAltoFirewallParser.ThreatCategory, "code-execution");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "vulnerability");
+ expected.put(BasicPaloAltoFirewallParser.ThreatID, "Eicar File Detected(39040)");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2018/02/01 21:29:03");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "guest");
+ expected.put(BasicPaloAltoFirewallParser.TunnelId, "0");
+ expected.put(BasicPaloAltoFirewallParser.TunnelType, "N/A");
+ expected.put(BasicPaloAltoFirewallParser.Type, "THREAT");
+ expected.put(BasicPaloAltoFirewallParser.URL, "www.eicar.org/download/eicar.com");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ expected.put(BasicPaloAltoFirewallParser.URLIndex, "9");
+ expected.put(BasicPaloAltoFirewallParser.WFReportID, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH1, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH2, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH3, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+ expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1");
+ assertEquals(expected, actual);
+ }
+
+ public static final String TRAFFIC_80 = "1,2018/02/01 21:24:11,001606000007,TRAFFIC,end,1,2018/02/01 21:24:11,172.16.2.31,134.19.6.22,192.168.18.2,134.19.6.22,Outgoing,,,ssl,vsys1,guest,internet,ethernet1/2.2,ethernet1/1,test,2018/02/01 21:24:11,19468,1,41537,443,12211,443,0x40001c,tcp,allow,7936,1731,6205,24,2018/02/01 21:00:42,1395,computer-and-internet-info,0,62977478,0x0,172.16.0.0-172.31.255.255,United States,0,14,10,tcp-rst-from-client,0,0,0,0,,PAN1,from-policy,,,0,,0,,N/A";
+ @SuppressWarnings("unchecked")
+ @Test
+ public void testParseTraffic80() throws ParseException {
+ JSONObject actual = parser.parse(TRAFFIC_80.getBytes()).get(0);
- for (Object o : json.entrySet()) {
- Entry entry = (Entry) o;
- String key = (String) entry.getKey();
- String value = json.get(key).toString();
- Assert.assertNotNull(value);
- }
- }
+ JSONObject expected = new JSONObject();
+ expected.put(BasicPaloAltoFirewallParser.Action, "allow");
+ expected.put(BasicPaloAltoFirewallParser.ActionFlags, "0x0");
+ expected.put(BasicPaloAltoFirewallParser.ActionSource, "from-policy");
+ expected.put(BasicPaloAltoFirewallParser.Application, "ssl");
+ expected.put(BasicPaloAltoFirewallParser.Bytes, "7936");
+ expected.put(BasicPaloAltoFirewallParser.BytesReceived, "6205");
+ expected.put(BasicPaloAltoFirewallParser.BytesSent, "1731");
+ expected.put(BasicPaloAltoFirewallParser.Category, "computer-and-internet-info");
+ expected.put(BasicPaloAltoFirewallParser.ConfigVersion, "1");
+ expected.put(BasicPaloAltoFirewallParser.DestinationLocation, "United States");
+ expected.put(BasicPaloAltoFirewallParser.ElapsedTimeInSec, "1395");
+ expected.put(BasicPaloAltoFirewallParser.Flags, "0x40001c");
+ expected.put(BasicPaloAltoFirewallParser.SourceZone, "guest");
+ expected.put(BasicPaloAltoFirewallParser.InboundInterface, "ethernet1/2.2");
+ expected.put(BasicPaloAltoFirewallParser.DestinationAddress, "134.19.6.22");
+ expected.put(BasicPaloAltoFirewallParser.DestinationPort, "443");
+ expected.put(BasicPaloAltoFirewallParser.SourceAddress, "172.16.2.31");
+ expected.put(BasicPaloAltoFirewallParser.SourcePort, "41537");
+ expected.put(BasicPaloAltoFirewallParser.LogAction, "test");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationPort, "443");
+ expected.put(BasicPaloAltoFirewallParser.NATDestinationIP, "134.19.6.22");
+ expected.put(BasicPaloAltoFirewallParser.NATSourcePort, "12211");
+ expected.put(BasicPaloAltoFirewallParser.NATSourceIP, "192.168.18.2");
+ expected.put("original_string", TRAFFIC_80);
+ expected.put(BasicPaloAltoFirewallParser.OutboundInterface, "ethernet1/1");
+ expected.put(BasicPaloAltoFirewallParser.Packets, "24");
+ expected.put(BasicPaloAltoFirewallParser.PaloAltoDomain, "1");
+ expected.put(BasicPaloAltoFirewallParser.ParentSessionId, "0");
+ expected.put(BasicPaloAltoFirewallParser.ParserVersion, 80);
+ expected.put(BasicPaloAltoFirewallParser.PktsReceived, "10");
+ expected.put(BasicPaloAltoFirewallParser.PktsSent, "14");
+ expected.put(BasicPaloAltoFirewallParser.IPProtocol, "tcp");
+ expected.put(BasicPaloAltoFirewallParser.ReceiveTime, "2018/02/01 21:24:11");
+ expected.put(BasicPaloAltoFirewallParser.RepeatCount, "1");
+ expected.put(BasicPaloAltoFirewallParser.Rule, "Outgoing");
+ expected.put(BasicPaloAltoFirewallParser.Seqno, "62977478");
+ expected.put(BasicPaloAltoFirewallParser.SerialNum, "001606000007");
+ expected.put(BasicPaloAltoFirewallParser.EndReason, "tcp-rst-from-client");
+ expected.put(BasicPaloAltoFirewallParser.SessionID, "19468");
+ expected.put(BasicPaloAltoFirewallParser.SourceLocation, "172.16.0.0-172.31.255.255");
+ expected.put(BasicPaloAltoFirewallParser.StartTime, "2018/02/01 21:00:42");
+ expected.put(BasicPaloAltoFirewallParser.ThreatContentType, "end");
+ expected.put(BasicPaloAltoFirewallParser.GenerateTime, "2018/02/01 21:24:11");
+ expected.put("timestamp", actual.get("timestamp"));
+ expected.put(BasicPaloAltoFirewallParser.DestinationZone, "internet");
+ expected.put(BasicPaloAltoFirewallParser.TunnelId, "0");
+ expected.put(BasicPaloAltoFirewallParser.TunnelType, "N/A");
+ expected.put(BasicPaloAltoFirewallParser.Type, "TRAFFIC");
+ expected.put(BasicPaloAltoFirewallParser.VirtualSystem, "vsys1");
+ expected.put(BasicPaloAltoFirewallParser.DGH1, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH2, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH3, "0");
+ expected.put(BasicPaloAltoFirewallParser.DGH4, "0");
+ expected.put(BasicPaloAltoFirewallParser.DeviceName, "PAN1");
+ assertEquals(expected, actual);
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/5f08ba0b/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt b/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
deleted file mode 100644
index c58bcc8..0000000
--- a/metron-platform/metron-parsers/src/test/resources/logData/PaloAltoFirewallParserTest.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-<11>Jan 5 05:38:59 PAN1.exampleCustomer.com 1,2015/01/05 05:38:58,0006C110285,THREAT,vulnerability,1,2015/01/05 05:38:58,10.0.0.115,216.0.10.198,0.0.0.0,0.0.0.0,EX-Allow,example\\user.name,,web-browsing,vsys1,internal,external,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 05:38:58,12031,1,54180,80,0,0,0x80004000,tcp,reset-both,\"ad.aspx?f=300x250&id=12;tile=1;ord=67AF705D60B1119C0F18BEA336F9\",HTTP: IIS Denial Of Service Attempt(40019),any,high,client-to-server,347368099,0x0,10.0.0.0-10.255.255.255,US,0,,1200568889751109656,,
-<14>Jan 5 12:51:34 PAN1.exampleCustomer.com 1,2015/01/05 12:51:33,0011C103117,TRAFFIC,end,1,2015/01/05 12:51:33,10.0.0.39,10.1.0.163,0.0.0.0,0.0.0.0,EX-Allow,,example\\user.name,ms-ds-smb,vsys1,v_external,v_internal,ethernet1/2,ethernet1/1,LOG-Default,2015/01/05 12:51:33,33760927,1,52688,445,0,0,0x401a,tcp,allow,2229,1287,942,10,2015/01/05 12:51:01,30,any,0,17754932062,0x0,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,6,4
\ No newline at end of file
[34/50] [abbrv] metron git commit: METRON-1091 STELLAR Shell: Stand
Alone installation (ottobackwards) closes apache/metron#690
Posted by rm...@apache.org.
METRON-1091 STELLAR Shell: Stand Alone installation (ottobackwards) closes apache/metron#690
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/124becd1
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/124becd1
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/124becd1
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 124becd1a35d00e620d4eaed5d5d84c0abbe8ba9
Parents: 644e951
Author: ottobackwards <ot...@gmail.com>
Authored: Tue Jan 30 16:34:16 2018 -0500
Committer: otto <ot...@apache.org>
Committed: Tue Jan 30 16:34:16 2018 -0500
----------------------------------------------------------------------
metron-stellar/stellar-common/NOTICE | 6 +++
metron-stellar/stellar-common/README.md | 43 +++++++++++++++
metron-stellar/stellar-common/pom.xml | 1 +
.../src/main/assembly/assembly.xml | 2 +-
.../src/main/assembly/stand-alone-assembly.xml | 57 ++++++++++++++++++++
.../src/main/scripts/deployed/stellar | 36 +++++++++++++
.../src/main/scripts/stand-alone/stellar | 21 ++++++++
.../stellar-common/src/main/scripts/stellar | 36 -------------
8 files changed, 165 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/NOTICE
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/NOTICE b/metron-stellar/stellar-common/NOTICE
new file mode 100644
index 0000000..1a19f3d
--- /dev/null
+++ b/metron-stellar/stellar-common/NOTICE
@@ -0,0 +1,6 @@
+ Apache Metron
+ Copyright 2015-2018 The Apache Software Foundation
+
+ This product includes software developed at
+ The Apache Software Foundation (http://www.apache.org/).
+
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 98aea77..a2f1242 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -1422,6 +1422,49 @@ Please note that functions are loading lazily in the background and will be unav
%functions
ABS, APPEND_IF_MISSING, BIN, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, CHOMP, CHOP, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENDS_WITH, FILL_LEFT, FILL_RIGHT, FILTER, FORMAT, GET, GET_FIRST, GET_LAST, HLLP_ADD, HLLP_CARDINALITY, HLLP_INIT, HLLP_MERGE, IN_SUBNET, IS_DATE, IS_DOMAIN, IS_EMAIL, IS_EMPTY, IS_INTEGER, IS_IP, IS_URL, JOIN, LENGTH, LIST_ADD, MAAS_GET_ENDPOINT, MAAS_MODEL_APPLY, MAP, MAP_EXISTS, MAP_GET, MONTH, OUTLIER_MAD_ADD, OUTLIER_MAD_SCORE, OUTLIER_MAD_STATE_MERGE, PREPEND_IF_MISSING, PROFILE_FIXED, PROFILE_GET, PROFILE_WINDOW, PROTOCOL_TO_NAME, REDUCE, REGEXP_MATCH, SPLIT, STARTS_WITH, STATS_ADD, STATS_BIN, STATS_COUNT, STATS_GEOMETRIC_MEAN, STATS_INIT, STATS_KURTOSIS, STATS_MAX, STATS_MEAN, STATS_MERGE, STATS_MIN, STATS_PERCENTILE, STATS_POPULATION_VARIANCE, STATS_QUADRATIC_MEAN, STATS_SD, STATS_SKEWNESS, STATS_SUM, STATS_SUM_LOGS, STATS_SUM_SQUARES, STATS_VARIANCE, STRING_ENTROPY, SYS
TEM_ENV_GET, SYSTEM_PROPERTY_GET, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR
```
+## Stellar Shell Stand Alone
+
+The Stellar Shell is also packaged as a stand alone application. It can be unpacked on any supported
+operating system.
+
+> Only the base Stellar functions are available as packaged. Other functions, such as those in metron-profiler and metron-management are not available.
+
+
+```bash
+metron-stellar/stellar-common/target/stellar-common-0.4.3-stand-alone.tar.gz
+```
+
+When unpacked, the following structure will be created:
+
+```bash
+.
+├── bin
+│ └── stellar
+└── lib
+ └── stellar-common-0.4.3-uber.jar
+```
+
+To run the Stellar Shell run the following from the directory you unpacked to:
+
+```bash
+bin/stellar
+```
+
+```bash
+-> % bin/stellar
+Stellar, Go!
+Please note that functions are loading lazily in the background and will be unavailable until loaded fully.
+[Stellar]>>> Functions loaded, you may refer to functions now...
+
+[Stellar]>>> %functions
+ABS, APPEND_IF_MISSING, BLOOM_ADD, BLOOM_EXISTS, BLOOM_INIT, BLOOM_MERGE, CEILING, CHOMP, CHOP, COS, COUNT_MATCHES, DAY_OF_MONTH, DAY_OF_WEEK, DAY_OF_YEAR, DECODE, DOMAIN_REMOVE_SUBDOMAINS, DOMAIN_REMOVE_TLD, DOMAIN_TO_TLD, ENCODE, ENDS_WITH, EXP, FILL_LEFT, FILL_RIGHT, FILTER, FLOOR, FORMAT, GET, GET_FIRST, GET_LAST, GET_SUPPORTED_ENCODINGS, IN_SUBNET, IS_EMPTY, IS_ENCODING, JOIN, LENGTH, LIST_ADD, LN, LOG10, LOG2, MAP, MAP_EXISTS, MAP_GET, MONTH, PREPEND_IF_MISSING, REDUCE, REGEXP_GROUP_VAL, REGEXP_MATCH, ROUND, SIN, SPLIT, SQRT, STARTS_WITH, STRING_ENTROPY, SYSTEM_ENV_GET, SYSTEM_PROPERTY_GET, TAN, TO_DOUBLE, TO_EPOCH_TIMESTAMP, TO_FLOAT, TO_INTEGER, TO_LONG, TO_LOWER, TO_STRING, TO_UPPER, TRIM, URL_TO_HOST, URL_TO_PATH, URL_TO_PORT, URL_TO_PROTOCOL, WEEK_OF_MONTH, WEEK_OF_YEAR, YEAR, ZIP, ZIP_LONGEST
+[Stellar]>>>
+```
+
+By default the shell will have the base Stellar Language commands available. Any jars in the lib directory
+that contain Stellar functions will also be loaded, and their commands will be available to shell, as long
+as their dependencies are satisfied.
+
### Implementation
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index a5dd20b..b43fcc1 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -348,6 +348,7 @@
<artifactId>maven-assembly-plugin</artifactId>
<configuration>
<descriptor>src/main/assembly/assembly.xml</descriptor>
+ <descriptor>src/main/assembly/stand-alone-assembly.xml</descriptor>
</configuration>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/src/main/assembly/assembly.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/assembly/assembly.xml b/metron-stellar/stellar-common/src/main/assembly/assembly.xml
index 0b5dfb8..b50be00 100644
--- a/metron-stellar/stellar-common/src/main/assembly/assembly.xml
+++ b/metron-stellar/stellar-common/src/main/assembly/assembly.xml
@@ -19,7 +19,7 @@
<includeBaseDirectory>false</includeBaseDirectory>
<fileSets>
<fileSet>
- <directory>${project.basedir}/src/main/scripts</directory>
+ <directory>${project.basedir}/src/main/scripts/deployed</directory>
<outputDirectory>bin</outputDirectory>
<useDefaultExcludes>true</useDefaultExcludes>
<excludes>
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/src/main/assembly/stand-alone-assembly.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/assembly/stand-alone-assembly.xml b/metron-stellar/stellar-common/src/main/assembly/stand-alone-assembly.xml
new file mode 100644
index 0000000..490f02e
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/assembly/stand-alone-assembly.xml
@@ -0,0 +1,57 @@
+<!--
+ Licensed to the Apache Software
+ Foundation (ASF) under one or more contributor license agreements. See the
+ NOTICE file distributed with this work for additional information regarding
+ copyright ownership. The ASF licenses this file to You under the Apache License,
+ Version 2.0 (the "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software distributed
+ under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES
+ OR CONDITIONS OF ANY KIND, either express or implied. See the License for
+ the specific language governing permissions and limitations under the License.
+ -->
+
+<assembly>
+ <id>stand-alone</id>
+ <formats>
+ <format>tar.gz</format>
+ </formats>
+ <includeBaseDirectory>false</includeBaseDirectory>
+ <fileSets>
+ <fileSet>
+ <directory>${project.basedir}/src/main/scripts/stand-alone</directory>
+ <outputDirectory>bin</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ <excludes>
+ <exclude>**/*.formatted</exclude>
+ <exclude>**/*.filtered</exclude>
+ </excludes>
+ <fileMode>0755</fileMode>
+ <lineEnding>unix</lineEnding>
+ <filtered>true</filtered>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/target</directory>
+ <includes>
+ <include>${project.artifactId}-${project.version}-uber.jar</include>
+ </includes>
+ <outputDirectory>lib</outputDirectory>
+ <useDefaultExcludes>true</useDefaultExcludes>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}</directory>
+ <includes>
+ <include>README.md</include>
+ </includes>
+ <outputDirectory></outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${project.basedir}/src/main/resources/META-INF</directory>
+ <includes>
+ <include>LICENSE</include>
+ <include>NOTICE</include>
+ </includes>
+ <outputDirectory></outputDirectory>
+ </fileSet>
+ </fileSets>
+</assembly>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/src/main/scripts/deployed/stellar
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/scripts/deployed/stellar b/metron-stellar/stellar-common/src/main/scripts/deployed/stellar
new file mode 100644
index 0000000..7b0f06d
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/scripts/deployed/stellar
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
+[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
+
+# Autodetect JAVA_HOME if not defined
+if [ -e /usr/libexec/bigtop-detect-javahome ]; then
+ . /usr/libexec/bigtop-detect-javahome
+elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
+ . /usr/lib/bigtop-utils/bigtop-detect-javahome
+fi
+
+export HBASE_CONFIGS=$(hbase classpath)
+export METRON_VERSION=${project.version}
+export METRON_HOME=/usr/metron/$METRON_VERSION
+export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
+export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
+java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/src/main/scripts/stand-alone/stellar
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/scripts/stand-alone/stellar b/metron-stellar/stellar-common/src/main/scripts/stand-alone/stellar
new file mode 100644
index 0000000..7906a32
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/scripts/stand-alone/stellar
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+binDir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+java $JVMFLAGS -cp "${binDir}/../lib/*" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
http://git-wip-us.apache.org/repos/asf/metron/blob/124becd1/metron-stellar/stellar-common/src/main/scripts/stellar
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/scripts/stellar b/metron-stellar/stellar-common/src/main/scripts/stellar
deleted file mode 100644
index 7b0f06d..0000000
--- a/metron-stellar/stellar-common/src/main/scripts/stellar
+++ /dev/null
@@ -1,36 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-BIGTOP_DEFAULTS_DIR=${BIGTOP_DEFAULTS_DIR-/etc/default}
-[ -n "${BIGTOP_DEFAULTS_DIR}" -a -r ${BIGTOP_DEFAULTS_DIR}/hbase ] && . ${BIGTOP_DEFAULTS_DIR}/hbase
-
-# Autodetect JAVA_HOME if not defined
-if [ -e /usr/libexec/bigtop-detect-javahome ]; then
- . /usr/libexec/bigtop-detect-javahome
-elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then
- . /usr/lib/bigtop-utils/bigtop-detect-javahome
-fi
-
-export HBASE_CONFIGS=$(hbase classpath)
-export METRON_VERSION=${project.version}
-export METRON_HOME=/usr/metron/$METRON_VERSION
-export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar)
-export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar)
-java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib}:$STELLAR_LIB:$MANAGEMENT_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@"
[37/50] [abbrv] metron git commit: METRON-1438 STELLAR: Move shell
functions to common from metron-management (ottobackwards) closes
apache/metron#920
Posted by rm...@apache.org.
METRON-1438 STELLAR: Move shell functions to common from metron-management (ottobackwards) closes apache/metron#920
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/c26abbba
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/c26abbba
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/c26abbba
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: c26abbbaeaeea3551e218cb9aa8ba97b32655958
Parents: 3d3c43c
Author: ottobackwards <ot...@gmail.com>
Authored: Fri Feb 2 09:58:59 2018 -0500
Committer: otto <ot...@apache.org>
Committed: Fri Feb 2 09:58:59 2018 -0500
----------------------------------------------------------------------
metron-platform/metron-management/README.md | 32 +-
.../metron/management/ShellFunctions.java | 302 -------------------
.../metron/management/ShellFunctionsTest.java | 171 -----------
metron-stellar/stellar-common/README.md | 31 ++
metron-stellar/stellar-common/pom.xml | 5 +
.../stellar/common/shell/cli/PausableInput.java | 23 +-
.../stellar/dsl/functions/ShellFunctions.java | 301 ++++++++++++++++++
.../dsl/functions/ShellFunctionsTest.java | 176 +++++++++++
8 files changed, 529 insertions(+), 512 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md
index 812583c..bf939c2 100644
--- a/metron-platform/metron-management/README.md
+++ b/metron-platform/metron-management/README.md
@@ -35,14 +35,13 @@ project.
* [Functions](#functions)
* [Grok Functions](#grok-functions)
* [File Functions](#file-functions)
- * [Shell Functions](#shell-functions)
* [Configuration Functions](#configuration-functions)
* [Parser Functions](#parser-functions)
* [Indexing Functions](#indexing-functions)
* [Enrichment Functions](#enrichment-functions)
* [Threat Triage Functions](#threat-triage-functions)
* [Examples](#examples)
- * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern)
+ * [Iterate to Find a Valid Grok Pattern](#iterate-to-find-a-valid-grok-pattern)
* [Manage Stellar Field Transformations](#manage-stellar-field-transformations)
* [Manage Stellar Enrichments](#manage-stellar-enrichments)
* [Manage Threat Triage Rules](#manage-threat-triage-rules)
@@ -132,35 +131,6 @@ The functions are split roughly into a few sections:
* path - The path of the file
* Returns: true if the file was written and false otherwise.
-### Shell Functions
-
-* `SHELL_EDIT`
- * Description: Open an editor (optionally initialized with text) and return whatever is saved from the editor. The editor to use is pulled from `EDITOR` or `VISUAL` environment variable.
- * Input:
- * string - (Optional) A string whose content is used to initialize the editor.
- * Returns: The content that the editor saved after editor exit.
-* `SHELL_GET_EXPRESSION`
- * Description: Get a stellar expression from a variable
- * Input:
- * variable - variable name
- * Returns: The stellar expression associated with the variable.
-* `SHELL_LIST_VARS`
- * Description: Return the variables in a tabular form
- * Input:
- * wrap : Length of string to wrap the columns
- * Returns: A tabular representation of the variables.
-* `SHELL_MAP2TABLE`
- * Description: Take a map and return a table
- * Input:
- * map - Map
- * Returns: The map in table form
-* `SHELL_VARS2MAP`
- * Description: Take a set of variables and return a map
- * Input:
- * variables* - variable names to use to create map
- * Returns: A map associating the variable name with the stellar expression.
-
-
### Configuration Functions
* `CONFIG_GET`
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java b/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
deleted file mode 100644
index afac7f0..0000000
--- a/metron-platform/metron-management/src/main/java/org/apache/metron/management/ShellFunctions.java
+++ /dev/null
@@ -1,302 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.management;
-
-import com.jakewharton.fliptables.FlipTable;
-import org.apache.commons.io.IOUtils;
-import org.apache.commons.lang3.text.WordUtils;
-import org.apache.metron.stellar.common.shell.VariableResult;
-import org.apache.metron.stellar.common.shell.cli.PausableInput;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
-import org.apache.metron.stellar.dsl.BaseStellarFunction;
-import org.apache.metron.stellar.dsl.Context;
-import org.apache.metron.stellar.dsl.ParseException;
-import org.apache.metron.stellar.dsl.Stellar;
-import org.apache.metron.stellar.dsl.StellarFunction;
-import org.jboss.aesh.console.Console;
-import org.slf4j.LoggerFactory;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.lang.invoke.MethodHandles;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE;
-
-public class ShellFunctions {
- private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static Map<String, VariableResult> getVariables(Context context) {
- return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get();
- }
-
- @Stellar(
- namespace = "SHELL"
- ,name = "MAP2TABLE"
- ,description = "Take a map and return a table"
- ,params = {"map - Map"
- }
- ,returns = "The map in table form"
- )
- public static class Map2Table extends BaseStellarFunction {
-
- @Override
- public Object apply(List<Object> args) {
- if(args.size() < 1) {
- return null;
- }
- Map<Object, Object> map = (Map<Object, Object>) args.get(0);
- if(map == null) {
- map = new HashMap<>();
- }
- String[] headers = {"KEY", "VALUE"};
- String[][] data = new String[map.size()][2];
- int i = 0;
- for(Map.Entry<Object, Object> kv : map.entrySet()) {
- data[i++] = new String[] {kv.getKey().toString(), kv.getValue().toString()};
- }
- return FlipTable.of(headers, data);
- }
- }
-
- @Stellar(
- namespace = "SHELL"
- ,name = "LIST_VARS"
- ,description = "Return the variables in a tabular form"
- ,params = {
- "wrap : Length of string to wrap the columns"
- }
- ,returns = "A tabular representation of the variables."
- )
- public static class ListVars implements StellarFunction {
-
- @Override
- public Object apply(List<Object> args, Context context) throws ParseException {
-
- Map<String, VariableResult> variables = getVariables(context);
- String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"};
- String[][] data = new String[variables.size()][3];
- int wordWrap = -1;
- if(args.size() > 0) {
- wordWrap = ConversionUtils.convert(args.get(0), Integer.class);
- }
- int i = 0;
- for(Map.Entry<String, VariableResult> kv : variables.entrySet()) {
- VariableResult result = kv.getValue();
- data[i++] = new String[] { toWrappedString(kv.getKey().toString(), wordWrap)
- , toWrappedString(result.getResult(), wordWrap)
- , toWrappedString(result.getExpression().get(), wordWrap)
- };
- }
- return FlipTable.of(headers, data);
- }
-
- private static String toWrappedString(Object o, int wrap) {
- String s = "" + o;
- if(wrap <= 0) {
- return s;
- }
- return WordUtils.wrap(s, wrap);
- }
-
- @Override
- public void initialize(Context context) {
-
- }
-
- @Override
- public boolean isInitialized() {
- return true;
- }
- }
-
- @Stellar(
- namespace = "SHELL"
- ,name = "VARS2MAP"
- ,description = "Take a set of variables and return a map"
- ,params = {"variables* - variable names to use to create map "
- }
- ,returns = "A map associating the variable name with the stellar expression."
- )
- public static class Var2Map implements StellarFunction {
-
- @Override
- public Object apply(List<Object> args, Context context) throws ParseException {
- Map<String, VariableResult> variables = getVariables(context);
- LinkedHashMap<String, String> ret = new LinkedHashMap<>();
- for(Object arg : args) {
- if(arg == null) {
- continue;
- }
- String variable = (String)arg;
- VariableResult result = variables.get(variable);
- if(result != null && result.getExpression().isPresent()) {
- ret.put(variable, result.getExpression().orElseGet(() -> ""));
- }
- }
- return ret;
- }
-
- @Override
- public void initialize(Context context) {
-
- }
-
- @Override
- public boolean isInitialized() {
- return true;
- }
- }
-
- @Stellar(
- namespace = "SHELL"
- ,name = "GET_EXPRESSION"
- ,description = "Get a stellar expression from a variable"
- ,params = {"variable - variable name"
- }
- ,returns = "The stellar expression associated with the variable."
- )
- public static class GetExpression implements StellarFunction {
-
- @Override
- public Object apply(List<Object> args, Context context) throws ParseException {
- Map<String, VariableResult> variables = getVariables(context);
- if(args.size() == 0) {
- return null;
- }
- String variable = (String) args.get(0);
- if(variable == null) {
- return null;
- }
- VariableResult result = variables.get(variable);
- if(result != null && result.getExpression().isPresent()) {
- return result.getExpression().get();
- }
- return null;
- }
-
- @Override
- public void initialize(Context context) {
-
- }
-
- @Override
- public boolean isInitialized() {
- return true;
- }
- }
-
- @Stellar(
- namespace = "SHELL"
- ,name = "EDIT"
- ,description = "Open an editor (optionally initialized with text) and return " +
- "whatever is saved from the editor. The editor to use is pulled " +
- "from `EDITOR` or `VISUAL` environment variable."
- ,params = { "string - (Optional) A string whose content is used to initialize the editor."
- }
- ,returns = "The content that the editor saved after editor exit."
- )
- public static class Edit implements StellarFunction {
-
- private String getEditor() {
- // if we have editor in the system properties, it should
- // override the env so we check that first
- String editor = System.getProperty("EDITOR");
- if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
- editor = System.getenv().get("EDITOR");
- }
- if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
- editor = System.getenv("VISUAL");
- }
- if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
- editor = "/bin/vi";
- }
- return editor;
- }
-
- @Override
- public Object apply(List<Object> args, Context context) throws ParseException {
- File outFile = null;
- String editor = getEditor();
- try {
- outFile = File.createTempFile("stellar_shell", "out");
- if(args.size() > 0) {
- String arg = (String)args.get(0);
- try(PrintWriter pw = new PrintWriter(outFile)) {
- IOUtils.write(arg, pw);
- }
- }
- } catch (IOException e) {
- String message = "Unable to create temp file: " + e.getMessage();
- LOG.error(message, e);
- throw new IllegalStateException(message, e);
- }
- Optional<Object> console = context.getCapability(CONSOLE, false);
- try {
- PausableInput.INSTANCE.pause();
- //shut down the IO for the console
- ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath());
- processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT);
- processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
- processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
- try {
- Process p = processBuilder.start();
- // wait for termination.
- p.waitFor();
- try (BufferedReader br = new BufferedReader(new FileReader(outFile))) {
- String ret = IOUtils.toString(br).trim();
- return ret;
- }
- } catch (Exception e) {
- String message = "Unable to read output: " + e.getMessage();
- LOG.error(message, e);
- return null;
- }
- } finally {
- try {
- PausableInput.INSTANCE.unpause();
- if(console.isPresent()) {
- ((Console)console.get()).pushToInputStream("\b\n");
- }
- } catch (IOException e) {
- LOG.error("Unable to unpause: {}", e.getMessage(), e);
- }
- if(outFile.exists()) {
- outFile.delete();
- }
- }
- }
-
- @Override
- public void initialize(Context context) {
-
- }
-
- @Override
- public boolean isInitialized() {
- return true;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java b/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
deleted file mode 100644
index 83c2bce..0000000
--- a/metron-platform/metron-management/src/test/java/org/apache/metron/management/ShellFunctionsTest.java
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.metron.management;
-
-import com.google.common.collect.ImmutableMap;
-import org.adrianwalker.multilinestring.Multiline;
-import org.apache.metron.stellar.common.shell.VariableResult;
-import org.apache.metron.stellar.dsl.Context;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Optional;
-
-import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
-
-public class ShellFunctionsTest {
-
- Map<String, VariableResult> variables = ImmutableMap.of(
- "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"),
- "var2" , VariableResult.withValue("foo"),
- "var3" , VariableResult.withValue(null),
- "var4" , VariableResult.withExpression(null, "blah")
- );
-
- Context context = new Context.Builder()
- .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
- .build();
-/**
-╔══════════╤═══════╤════════════╗
-║ VARIABLE │ VALUE │ EXPRESSION ║
-╠══════════╪═══════╪════════════╣
-║ foo │ 2.0 │ 1 + 1 ║
-╚══════════╧═══════╧════════════╝
- **/
- @Multiline
- static String expectedListWithFoo;
-
- @Test
- public void testListVarsWithVars() {
- Map<String, VariableResult> variables = ImmutableMap.of(
- "foo", VariableResult.withExpression(2.0, "1 + 1"));
-
- Context context = new Context.Builder()
- .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
- .build();
- Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
- Assert.assertEquals(expectedListWithFoo, out);
- }
-
-/**
-╔══════════╤═══════╤════════════╗
-║ VARIABLE │ VALUE │ EXPRESSION ║
-╠══════════╧═══════╧════════════╣
-║ (empty) ║
-╚═══════════════════════════════╝
- **/
- @Multiline
- static String expectedEmptyList;
-
- @Test
- public void testListVarsWithoutVars() {
- Context context = new Context.Builder()
- .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>())
- .build();
- Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
- Assert.assertEquals(expectedEmptyList, out);
- }
-/**
-╔════════╤═══════╗
-║ KEY │ VALUE ║
-╠════════╪═══════╣
-║ field1 │ val1 ║
-╟────────┼───────╢
-║ field2 │ val2 ║
-╚════════╧═══════╝
- **/
- @Multiline
- static String expectedMap2Table;
-
- @Test
- public void testMap2Table() {
- Map<String, Object> variables = ImmutableMap.of("map_field", ImmutableMap.of("field1", "val1", "field2", "val2"));
- Context context = Context.EMPTY_CONTEXT();
- Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
- Assert.assertEquals(expectedMap2Table, out);
- }
- /**
-╔═════╤═══════╗
-║ KEY │ VALUE ║
-╠═════╧═══════╣
-║ (empty) ║
-╚═════════════╝
- **/
- @Multiline
- static String expectedMap2TableNullInput;
-
- @Test
- public void testMap2TableNullInput() {
- Map<String,Object> variables = new HashMap<String,Object>(){{
- put("map_field",null);
- }};
- Context context = Context.EMPTY_CONTEXT();
- Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
- Assert.assertEquals(expectedMap2TableNullInput, out);
- }
-
- @Test
- public void testMap2TableInsufficientArgs() {
- Map<String, Object> variables = new HashMap<>();
- Context context = Context.EMPTY_CONTEXT();
- Object out = run("SHELL_MAP2TABLE()", variables, context);
- Assert.assertNull(out);
- }
-
- @Test
- public void testVars2Map() {
- Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), context);
- Assert.assertTrue(out instanceof Map);
- Map<String, String> mapOut = (Map<String, String>)out;
- //second one is null, so we don't want it there.
- Assert.assertEquals(1, mapOut.size());
- Assert.assertEquals("TO_UPPER('casey')", mapOut.get("var1"));
- }
-
- @Test
- public void testVars2MapEmpty() {
- Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context);
- Map<String, String> mapOut = (Map<String, String>)out;
- Assert.assertEquals(0, mapOut.size());
- }
-
- @Test
- public void testGetExpression() {
- Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context);
- Assert.assertTrue(out instanceof String);
- String expression = (String)out;
- //second one is null, so we don't want it there.
- Assert.assertEquals("TO_UPPER('casey')", expression);
- }
-
- @Test
- public void testGetExpressionEmpty() {
- Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context);
- Assert.assertNull(out );
- }
-
- @Test
- public void testEdit() throws Exception {
- System.getProperties().put("EDITOR", "/bin/cat");
- Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", "foo"), context);
- Assert.assertEquals("FOO", out);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index a2f1242..078799e 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -243,6 +243,11 @@ Where:
| [ `SET_MERGE`](#set_merge) |
| [ `SET_REMOVE`](#set_remove) |
| [ `SIN`](#sin) |
+| [ `SHELL_EDIT`](#shell_edit) |
+| [ `SHELL_GET_EXPRESSION`](#shell_get_expression) |
+| [ `SHELL_LIST_VARS`](#shell_list_vars) |
+| [ `SHELL_MAP2TABLE`](#shell_map2table) |
+| [ `SHELL_VARS2MAP`](#shell_vars2map) |
| [ `SPLIT`](#split) |
| [ `SQRT`](#sqrt) |
| [ `STARTS_WITH`](#starts_with) |
@@ -920,6 +925,32 @@ Where:
* o - object to add to set
* Returns: A Set
+### `SHELL_EDIT`
+ * Description: Open an editor (optionally initialized with text) and return whatever is saved from the editor. The editor to use is pulled from `EDITOR` or `VISUAL` environment variable.
+ * Input:
+ * string - (Optional) A string whose content is used to initialize the editor.
+ * Returns: The content that the editor saved after editor exit.
+### `SHELL_GET_EXPRESSION`
+ * Description: Get a stellar expression from a variable
+ * Input:
+ * variable - variable name
+ * Returns: The stellar expression associated with the variable.
+### `SHELL_LIST_VARS`
+ * Description: Return the variables in a tabular form
+ * Input:
+ * wrap : Length of string to wrap the columns
+ * Returns: A tabular representation of the variables.
+### `SHELL_MAP2TABLE`
+ * Description: Take a map and return a table
+ * Input:
+ * map - Map
+ * Returns: The map in table form
+### `SHELL_VARS2MAP`
+ * Description: Take a set of variables and return a map
+ * Input:
+ * variables* - variable names to use to create map
+ * Returns: A map associating the variable name with the stellar expression.
+
### `SIN`
* Description: Returns the sine of a number.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/pom.xml
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/pom.xml b/metron-stellar/stellar-common/pom.xml
index b43fcc1..6b07e68 100644
--- a/metron-stellar/stellar-common/pom.xml
+++ b/metron-stellar/stellar-common/pom.xml
@@ -203,6 +203,11 @@
<version>${global_kryo_serializers_version}</version>
</dependency>
<dependency>
+ <groupId>com.jakewharton.fliptables</groupId>
+ <artifactId>fliptables</artifactId>
+ <version>1.0.2</version>
+ </dependency>
+ <dependency>
<!-- junit dependency added with default scope to allow inclusion of StellarProcessorUtils in main jar.
It is excluded from the uber-jar. -->
<groupId>junit</groupId>
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
index fad0115..c72d66f 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/common/shell/cli/PausableInput.java
@@ -21,6 +21,7 @@ package org.apache.metron.stellar.common.shell.cli;
import java.io.IOException;
import java.io.InputStream;
+import java.util.concurrent.atomic.AtomicBoolean;
/**
* An input stream which mirrors System.in, but allows you to 'pause' and 'unpause' it.
@@ -36,8 +37,8 @@ import java.io.InputStream;
*
*/
public class PausableInput extends InputStream {
- InputStream in = System.in;
- boolean paused = false;
+ private InputStream in = System.in;
+ private AtomicBoolean paused = new AtomicBoolean(false);
private PausableInput() {
super();
}
@@ -46,7 +47,7 @@ public class PausableInput extends InputStream {
* Stop mirroring stdin
*/
public void pause() {
- paused = true;
+ paused.set(true);
}
/**
@@ -54,8 +55,7 @@ public class PausableInput extends InputStream {
* @throws IOException
*/
public void unpause() throws IOException {
- in.read(new byte[in.available()]);
- paused = false;
+ paused.set(false);
}
public final static PausableInput INSTANCE = new PausableInput();
@@ -76,7 +76,14 @@ public class PausableInput extends InputStream {
*/
@Override
public int read() throws IOException {
-
+ if(paused.get()) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ return 0;
+ }
return in.read();
}
@@ -116,7 +123,7 @@ public class PausableInput extends InputStream {
@Override
public int read(byte[] b) throws IOException {
- if(paused) {
+ if(paused.get()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
@@ -187,7 +194,7 @@ public class PausableInput extends InputStream {
*/
@Override
public int read(byte[] b, int off, int len) throws IOException {
- if(paused) {
+ if(paused.get()) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
new file mode 100644
index 0000000..1df4a51
--- /dev/null
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/ShellFunctions.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+import static org.apache.metron.stellar.dsl.Context.Capabilities.CONSOLE;
+
+import com.jakewharton.fliptables.FlipTable;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.lang.invoke.MethodHandles;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang3.text.WordUtils;
+import org.apache.metron.stellar.common.shell.VariableResult;
+import org.apache.metron.stellar.common.shell.cli.PausableInput;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
+import org.apache.metron.stellar.dsl.BaseStellarFunction;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.ParseException;
+import org.apache.metron.stellar.dsl.Stellar;
+import org.apache.metron.stellar.dsl.StellarFunction;
+import org.jboss.aesh.console.Console;
+import org.slf4j.LoggerFactory;
+
+public class ShellFunctions {
+ private static final org.slf4j.Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, VariableResult> getVariables(Context context) {
+ return (Map<String, VariableResult>) context.getCapability(Context.Capabilities.SHELL_VARIABLES).get();
+ }
+
+ @Stellar(
+ namespace = "SHELL"
+ ,name = "MAP2TABLE"
+ ,description = "Take a map and return a table"
+ ,params = {"map - Map"
+ }
+ ,returns = "The map in table form"
+ )
+ public static class Map2Table extends BaseStellarFunction {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public Object apply(List<Object> args) {
+ if (args.size() < 1) {
+ return null;
+ }
+ Map<Object, Object> map = (Map<Object, Object>) args.get(0);
+ if (map == null) {
+ map = new HashMap<>();
+ }
+ String[] headers = {"KEY", "VALUE"};
+ String[][] data = new String[map.size()][2];
+ int i = 0;
+ for (Map.Entry<Object, Object> kv : map.entrySet()) {
+ data[i++] = new String[]{kv.getKey().toString(), kv.getValue().toString()};
+ }
+ return FlipTable.of(headers, data);
+ }
+ }
+
+ @Stellar(
+ namespace = "SHELL"
+ ,name = "LIST_VARS"
+ ,description = "Return the variables in a tabular form"
+ ,params = {
+ "wrap : Length of string to wrap the columns"
+ }
+ ,returns = "A tabular representation of the variables."
+ )
+ public static class ListVars implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ Map<String, VariableResult> variables = getVariables(context);
+ String[] headers = {"VARIABLE", "VALUE", "EXPRESSION"};
+ String[][] data = new String[variables.size()][3];
+ int wordWrap = -1;
+ if (args.size() > 0) {
+ wordWrap = ConversionUtils.convert(args.get(0), Integer.class);
+ }
+ int i = 0;
+ for (Map.Entry<String, VariableResult> kv : variables.entrySet()) {
+ VariableResult result = kv.getValue();
+ data[i++] = new String[]{toWrappedString(kv.getKey(), wordWrap),
+ toWrappedString(result.getResult(), wordWrap),
+ toWrappedString(result.getExpression().get(), wordWrap)};
+ }
+ return FlipTable.of(headers, data);
+ }
+
+ private static String toWrappedString(Object o, int wrap) {
+ String s = "" + o;
+ if(wrap <= 0) {
+ return s;
+ }
+ return WordUtils.wrap(s, wrap);
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(
+ namespace = "SHELL"
+ ,name = "VARS2MAP"
+ ,description = "Take a set of variables and return a map"
+ ,params = {"variables* - variable names to use to create map "
+ }
+ ,returns = "A map associating the variable name with the stellar expression."
+ )
+ public static class Var2Map implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ Map<String, VariableResult> variables = getVariables(context);
+ LinkedHashMap<String, String> ret = new LinkedHashMap<>();
+ for (Object arg : args) {
+ if (arg == null) {
+ continue;
+ }
+ String variable = (String) arg;
+ VariableResult result = variables.get(variable);
+ if (result != null && result.getExpression().isPresent()) {
+ ret.put(variable, result.getExpression().orElseGet(() -> ""));
+ }
+ }
+ return ret;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(
+ namespace = "SHELL"
+ ,name = "GET_EXPRESSION"
+ ,description = "Get a stellar expression from a variable"
+ ,params = {"variable - variable name"
+ }
+ ,returns = "The stellar expression associated with the variable."
+ )
+ public static class GetExpression implements StellarFunction {
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ Map<String, VariableResult> variables = getVariables(context);
+ if (args.size() == 0) {
+ return null;
+ }
+ String variable = (String) args.get(0);
+ if (variable == null) {
+ return null;
+ }
+ VariableResult result = variables.get(variable);
+ if (result != null && result.getExpression().isPresent()) {
+ return result.getExpression().get();
+ }
+ return null;
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+
+ @Stellar(
+ namespace = "SHELL"
+ ,name = "EDIT"
+ ,description = "Open an editor (optionally initialized with text) and return " +
+ "whatever is saved from the editor. The editor to use is pulled " +
+ "from `EDITOR` or `VISUAL` environment variable."
+ ,params = { "string - (Optional) A string whose content is used to initialize the editor."
+ }
+ ,returns = "The content that the editor saved after editor exit."
+ )
+ public static class Edit implements StellarFunction {
+
+ private String getEditor() {
+ // if we have editor in the system properties, it should
+ // override the env so we check that first
+ String editor = System.getProperty("EDITOR");
+ if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
+ editor = System.getenv().get("EDITOR");
+ }
+ if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
+ editor = System.getenv("VISUAL");
+ }
+ if(org.apache.commons.lang3.StringUtils.isEmpty(editor)) {
+ editor = "/bin/vi";
+ }
+ return editor;
+ }
+
+ @Override
+ public Object apply(List<Object> args, Context context) throws ParseException {
+ File outFile = null;
+ String editor = getEditor();
+ try {
+ outFile = File.createTempFile("stellar_shell", "out");
+ if (args.size() > 0) {
+ String arg = (String) args.get(0);
+ try (PrintWriter pw = new PrintWriter(outFile)) {
+ IOUtils.write(arg, pw);
+ }
+ }
+ } catch (IOException e) {
+ String message = "Unable to create temp file: " + e.getMessage();
+ LOG.error(message, e);
+ throw new IllegalStateException(message, e);
+ }
+ Optional<Object> console = context.getCapability(CONSOLE, false);
+ try {
+ PausableInput.INSTANCE.pause();
+ //shut down the IO for the console
+ ProcessBuilder processBuilder = new ProcessBuilder(editor, outFile.getAbsolutePath());
+ processBuilder.redirectInput(ProcessBuilder.Redirect.INHERIT);
+ processBuilder.redirectOutput(ProcessBuilder.Redirect.INHERIT);
+ processBuilder.redirectError(ProcessBuilder.Redirect.INHERIT);
+ try {
+ Process p = processBuilder.start();
+ // wait for termination.
+ p.waitFor();
+ try (BufferedReader br = new BufferedReader(new FileReader(outFile))) {
+ String ret = IOUtils.toString(br).trim();
+ return ret;
+ }
+ } catch (Exception e) {
+ String message = "Unable to read output: " + e.getMessage();
+ LOG.error(message, e);
+ return null;
+ }
+ } finally {
+ try {
+ PausableInput.INSTANCE.unpause();
+ if (console.isPresent()) {
+ ((Console) console.get()).pushToInputStream("\b\n");
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to unpause: {}", e.getMessage(), e);
+ }
+ if (outFile.exists()) {
+ outFile.delete();
+ }
+ }
+ }
+
+ @Override
+ public void initialize(Context context) {
+
+ }
+
+ @Override
+ public boolean isInitialized() {
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/c26abbba/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
new file mode 100644
index 0000000..354e0c3
--- /dev/null
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/ShellFunctionsTest.java
@@ -0,0 +1,176 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.metron.stellar.dsl.functions;
+
+import com.google.common.collect.ImmutableMap;
+import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.stellar.common.shell.VariableResult;
+import org.apache.metron.stellar.common.shell.cli.PausableInput;
+import org.apache.metron.stellar.dsl.Context;
+import org.apache.metron.stellar.dsl.Context.Capabilities;
+import org.jboss.aesh.console.Console;
+import org.jboss.aesh.console.settings.Settings;
+import org.jboss.aesh.console.settings.SettingsBuilder;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.metron.stellar.common.utils.StellarProcessorUtils.run;
+
+public class ShellFunctionsTest {
+
+ Map<String, VariableResult> variables = ImmutableMap.of(
+ "var1" , VariableResult.withExpression("CASEY", "TO_UPPER('casey')"),
+ "var2" , VariableResult.withValue("foo"),
+ "var3" , VariableResult.withValue(null),
+ "var4" , VariableResult.withExpression(null, "blah")
+ );
+
+ Context context = new Context.Builder()
+ .with(Context.Capabilities.SHELL_VARIABLES , () -> variables).build();
+
+/**
+╔══════════╤═══════╤════════════╗
+║ VARIABLE │ VALUE │ EXPRESSION ║
+╠══════════╪═══════╪════════════╣
+║ foo │ 2.0 │ 1 + 1 ║
+╚══════════╧═══════╧════════════╝
+ **/
+ @Multiline
+ static String expectedListWithFoo;
+
+ @Test
+ public void testListVarsWithVars() {
+ Map<String, VariableResult> variables = ImmutableMap.of(
+ "foo", VariableResult.withExpression(2.0, "1 + 1"));
+
+ Context context = new Context.Builder()
+ .with(Context.Capabilities.SHELL_VARIABLES , () -> variables)
+ .build();
+ Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
+ Assert.assertEquals(expectedListWithFoo, out);
+ }
+
+/**
+╔══════════╤═══════╤════════════╗
+║ VARIABLE │ VALUE │ EXPRESSION ║
+╠══════════╧═══════╧════════════╣
+║ (empty) ║
+╚═══════════════════════════════╝
+ **/
+ @Multiline
+ static String expectedEmptyList;
+
+ @Test
+ public void testListVarsWithoutVars() {
+ Context context = new Context.Builder()
+ .with(Context.Capabilities.SHELL_VARIABLES, () -> new HashMap<>())
+ .build();
+ Object out = run("SHELL_LIST_VARS()", new HashMap<>(), context);
+ Assert.assertEquals(expectedEmptyList, out);
+ }
+/**
+╔════════╤═══════╗
+║ KEY │ VALUE ║
+╠════════╪═══════╣
+║ field1 │ val1 ║
+╟────────┼───────╢
+║ field2 │ val2 ║
+╚════════╧═══════╝
+ **/
+ @Multiline
+ static String expectedMap2Table;
+
+ @Test
+ public void testMap2Table() {
+ Map<String, Object> variables = ImmutableMap.of("map_field", ImmutableMap.of("field1", "val1", "field2", "val2"));
+ Context context = Context.EMPTY_CONTEXT();
+ Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
+ Assert.assertEquals(expectedMap2Table, out);
+ }
+ /**
+╔═════╤═══════╗
+║ KEY │ VALUE ║
+╠═════╧═══════╣
+║ (empty) ║
+╚═════════════╝
+ **/
+ @Multiline
+ static String expectedMap2TableNullInput;
+
+ @Test
+ public void testMap2TableNullInput() {
+ Map<String,Object> variables = new HashMap<String,Object>(){{
+ put("map_field",null);
+ }};
+ Context context = Context.EMPTY_CONTEXT();
+ Object out = run("SHELL_MAP2TABLE(map_field)", variables, context);
+ Assert.assertEquals(expectedMap2TableNullInput, out);
+ }
+
+ @Test
+ public void testMap2TableInsufficientArgs() {
+ Map<String, Object> variables = new HashMap<>();
+ Context context = Context.EMPTY_CONTEXT();
+ Object out = run("SHELL_MAP2TABLE()", variables, context);
+ Assert.assertNull(out);
+ }
+
+ @Test
+ public void testVars2Map() {
+ Object out = run("SHELL_VARS2MAP('var1', 'var2')", new HashMap<>(), context);
+ Assert.assertTrue(out instanceof Map);
+ Map<String, String> mapOut = (Map<String, String>)out;
+ //second one is null, so we don't want it there.
+ Assert.assertEquals(1, mapOut.size());
+ Assert.assertEquals("TO_UPPER('casey')", mapOut.get("var1"));
+ }
+
+ @Test
+ public void testVars2MapEmpty() {
+ Object out = run("SHELL_VARS2MAP()", new HashMap<>(), context);
+ Map<String, String> mapOut = (Map<String, String>)out;
+ Assert.assertEquals(0, mapOut.size());
+ }
+
+ @Test
+ public void testGetExpression() {
+ Object out = run("SHELL_GET_EXPRESSION('var1')", new HashMap<>(), context);
+ Assert.assertTrue(out instanceof String);
+ String expression = (String)out;
+ //second one is null, so we don't want it there.
+ Assert.assertEquals("TO_UPPER('casey')", expression);
+ }
+
+ @Test
+ public void testGetExpressionEmpty() {
+ Object out = run("SHELL_GET_EXPRESSION()", new HashMap<>(), context);
+ Assert.assertNull(out );
+ }
+
+ @Test
+ public void testEdit() throws Exception {
+ System.getProperties().put("EDITOR", "/bin/cat");
+ Object out = run("TO_UPPER(SHELL_EDIT(foo))", ImmutableMap.of("foo", "foo"), context);
+ Assert.assertEquals("FOO", out);
+ }
+
+}
[04/50] [abbrv] metron git commit: METRON-1429 SearchIntegrationTest
refactor (merrimanr) closes apache/metron#909
Posted by rm...@apache.org.
METRON-1429 SearchIntegrationTest refactor (merrimanr) closes apache/metron#909
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/35d81cb9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/35d81cb9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/35d81cb9
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 35d81cb9d1e7cac9d13b64bc3baaf6f6b3354d82
Parents: acb8b92
Author: merrimanr <me...@gmail.com>
Authored: Thu Jan 25 10:30:06 2018 -0600
Committer: merrimanr <me...@apache.org>
Committed: Thu Jan 25 10:30:06 2018 -0600
----------------------------------------------------------------------
.../MetaAlertControllerIntegrationTest.java | 11 +-
.../UpdateControllerIntegrationTest.java | 2 +-
.../ElasticsearchSearchIntegrationTest.java | 99 +-----
.../indexing/dao/SearchIntegrationTest.java | 345 +++++--------------
4 files changed, 94 insertions(+), 363 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
index b0dd774..3e69e37 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/MetaAlertControllerIntegrationTest.java
@@ -96,13 +96,22 @@ public class MetaAlertControllerIntegrationTest extends DaoControllerTest {
@Multiline
public static String create;
+ /**
+ * [
+ *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
+ *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
+ * ]
+ */
+ @Multiline
+ public static String metaAlertData;
+
@Before
public void setup() throws Exception {
this.mockMvc = MockMvcBuilders.webAppContextSetup(this.wac).apply(springSecurity()).build();
ImmutableMap<String, String> testData = ImmutableMap.of(
"bro_index_2017.01.01.01", SearchIntegrationTest.broData,
"snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
- MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
+ MetaAlertDao.METAALERTS_INDEX, metaAlertData
);
loadTestData(testData);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
index 57a1b28..e8d00d3 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/UpdateControllerIntegrationTest.java
@@ -121,7 +121,7 @@ public class UpdateControllerIntegrationTest extends DaoControllerTest {
ImmutableMap<String, String> testData = ImmutableMap.of(
"bro_index_2017.01.01.01", SearchIntegrationTest.broData,
"snort_index_2017.01.01.01", SearchIntegrationTest.snortData,
- MetaAlertDao.METAALERTS_INDEX, SearchIntegrationTest.metaAlertData
+ MetaAlertDao.METAALERTS_INDEX, MetaAlertControllerIntegrationTest.metaAlertData
);
loadTestData(testData);
}
http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 3949c6d..f86a04d 100644
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@ -19,35 +19,25 @@ package org.apache.metron.elasticsearch.integration;
import java.io.File;
-import java.util.HashMap;
import java.io.IOException;
+import java.util.HashMap;
import java.util.concurrent.ExecutionException;
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
-import org.apache.metron.indexing.dao.MetaAlertDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
import org.apache.metron.integration.InMemoryComponent;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.support.WriteRequest;
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.search.SearchHit;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.concurrent.ExecutionException;
-
public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
private static String indexDir = "target/elasticsearch_search";
@@ -181,34 +171,6 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
@Multiline
private static String broDefaultStringMappings;
- /**
- * {
- * "metaalert_doc": {
- * "properties": {
- * "guid": { "type": "keyword" },
- * "alert": {
- * "type": "nested",
- * "properties": {
- * "guid": { "type": "keyword" }
- * }
- * },
- * "average": { "type": "keyword" },
- * "min" : { "type": "keyword" },
- * "median" : { "type": "keyword" },
- * "max": { "type": "keyword" },
- * "count": { "type": "keyword" },
- * "sum": { "type": "keyword" },
- * "source:type": {
- * "type": "text",
- * "fielddata" : "true"
- * }
- * }
- * }
- * }
- */
- @Multiline
- private static String metaAlertTypeMappings;
-
@Override
protected IndexDao createDao() throws Exception {
AccessConfig config = new AccessConfig();
@@ -246,14 +208,13 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
.addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get();
es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
.addMapping("snort_doc", snortTypeMappings).get();
- es.getClient().admin().indices().prepareCreate(MetaAlertDao.METAALERTS_INDEX)
- .addMapping(MetaAlertDao.METAALERT_DOC, metaAlertTypeMappings).get();
BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
for(Object o: broArray) {
JSONObject jsonObject = (JSONObject) o;
IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("bro_index_2017.01.01.01", "bro_doc");
+ indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
bulkRequest.add(indexRequestBuilder);
@@ -262,68 +223,14 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
for(Object o: snortArray) {
JSONObject jsonObject = (JSONObject) o;
IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("snort_index_2017.01.01.02", "snort_doc");
+ indexRequestBuilder = indexRequestBuilder.setId((String) jsonObject.get("guid"));
indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString());
bulkRequest.add(indexRequestBuilder);
}
- JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData);
- for(Object o: metaAlertArray) {
- JSONObject jsonObject = (JSONObject) o;
- IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalert_index", "metaalert_doc");
- indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString());
- bulkRequest.add(indexRequestBuilder);
- }
BulkResponse bulkResponse = bulkRequest.execute().actionGet();
if (bulkResponse.hasFailures()) {
throw new RuntimeException("Failed to index test data");
}
-
- SearchResponse broDocs = es.getClient()
- .prepareSearch("bro_index_2017.01.01.01")
- .setTypes("bro_doc")
- .setQuery(QueryBuilders.matchAllQuery())
- .get();
- // We're changing the _id field, we need to create a copy and delete the original.
- for (SearchHit hit : broDocs.getHits()) {
- // Bro GUIDs to collide while using the standard analyzer
- // Use timestamp as part of guid because query may not return in order each time
- IndexRequest indexRequest = new IndexRequest()
- .index("bro_index_2017.01.01.01")
- .type("bro_doc")
- .id("bro-" + hit.getSource().get("timestamp"))
- .source(hit.getSource());
- es.getClient().index(indexRequest).get();
-
- // Delete the original
- es.getClient()
- .prepareDelete("bro_index_2017.01.01.01", "bro_doc", hit.getId())
- .get();
- }
-
- // Wait until everything is updated
- // Assume true until proven otherwise.
- boolean allUpdated = true;
- for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
- allUpdated = true;
- SearchResponse response = es.getClient()
- .prepareSearch("bro_index_2017.01.01.01")
- .setTypes("bro_doc")
- .setQuery(QueryBuilders.matchAllQuery())
- .get();
- if (response.getHits().getTotalHits() == 0) {
- throw new IllegalStateException("Bro index is empty. No docs to validate were updated");
- }
- for (SearchHit hit : response.getHits()) {
- if (!hit.getId().startsWith("bro-")) {
- allUpdated = false;
- }
- }
- if (allUpdated) {
- break;
- }
- }
- if (!allUpdated) {
- throw new IllegalStateException("Unable to update Elasticsearch ids properly");
- }
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/35d81cb9/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index 72e632f..f381688 100644
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@ -46,14 +46,6 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-
public abstract class SearchIntegrationTest {
/**
* [
@@ -80,15 +72,6 @@ public abstract class SearchIntegrationTest {
public static String snortData;
/**
- * [
- *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
- *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
- * ]
- */
- @Multiline
- public static String metaAlertData;
-
- /**
* {
* "indices": ["bro", "snort"],
* "query": "*",
@@ -107,7 +90,7 @@ public abstract class SearchIntegrationTest {
/**
* {
- * "guid": "bro-3",
+ * "guid": "bro_3",
* "sensorType": "bro"
* }
*/
@@ -117,12 +100,12 @@ public abstract class SearchIntegrationTest {
/**
* [
* {
- * "guid": "bro-1",
+ * "guid": "bro_1",
* "sensorType": "bro"
* },
* {
- * "guid": "bro-2",
- * "sensorType": "bro"
+ * "guid": "snort_2",
+ * "sensorType": "snort"
* }
* ]
*/
@@ -240,7 +223,7 @@ public abstract class SearchIntegrationTest {
/**
* {
* "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"],
- * "indices": ["bro", "snort", "metaalert"],
+ * "indices": ["bro", "snort"],
* "query": "*",
* "from": 0,
* "size": 10,
@@ -346,7 +329,7 @@ public abstract class SearchIntegrationTest {
/**
* {
* "fields": ["guid"],
- * "indices": ["metaalert"],
+ * "indices": ["bro"],
* "query": "*",
* "from": 0,
* "size": 10,
@@ -360,7 +343,7 @@ public abstract class SearchIntegrationTest {
* }
*/
@Multiline
- public static String metaAlertsFieldQuery;
+ public static String sortByGuidQuery;
/**
* {
@@ -373,7 +356,7 @@ public abstract class SearchIntegrationTest {
* }
* ],
* "scoreField":"score",
- * "indices": ["bro", "snort", "metaalert"],
+ * "indices": ["bro", "snort"],
* "query": "*"
* }
*/
@@ -398,7 +381,7 @@ public abstract class SearchIntegrationTest {
* }
* }
* ],
- * "indices": ["bro", "snort", "metaalert"],
+ * "indices": ["bro", "snort"],
* "query": "*"
* }
*/
@@ -453,6 +436,23 @@ public abstract class SearchIntegrationTest {
public ExpectedException thrown = ExpectedException.none();
@Test
+ public void all_query_returns_all_results() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, results.size());
+ for(int i = 0;i < 5;++i) {
+ Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
+ Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
+ }
+ for (int i = 5; i < 10; ++i) {
+ Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
+ Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
+ }
+ }
+
+ @Test
public void find_one_guid() throws Exception {
GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
Optional<Map<String, Object>> response = dao.getLatestResult(request);
@@ -463,240 +463,19 @@ public abstract class SearchIntegrationTest {
}
@Test
- public void all_query_returns_all_results() throws Exception {
- //All Query Testcase
- {
- SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, results.size());
- for(int i = 0;i < 5;++i) {
- Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
- Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
- }
- for (int i = 5; i < 10; ++i) {
- Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
- Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
- }
- }
- //Get All Latest Guid Testcase
- {
- List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>() {
- });
- Map<String, Document> docs = new HashMap<>();
-
- for(Document doc : dao.getAllLatest(request)) {
- docs.put(doc.getGuid(), doc);
- }
- Assert.assertEquals(2, docs.size());
- Assert.assertTrue(docs.keySet().contains("bro-1"));
- Assert.assertTrue(docs.keySet().contains("bro-2"));
- for(Map.Entry<String, Document> kv : docs.entrySet()) {
- Document d = kv.getValue();
- Assert.assertEquals("bro", d.getDocument().get("source:type"));
- }
- }
- //Filter test case
- {
- SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(3, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
- Assert.assertEquals(9, results.get(0).getSource().get("timestamp"));
- Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
- Assert.assertEquals(7, results.get(1).getSource().get("timestamp"));
- Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
- Assert.assertEquals(1, results.get(2).getSource().get("timestamp"));
- }
- //Sort test case
- {
- SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- for(int i = 8001;i < 8011;++i) {
- Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port"));
- }
- }
- //Sort descending with missing fields
- {
- SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, results.size());
-
- // validate sorted order - there are only 2 with a 'threat:triage:score'
- Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
- Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
-
- // the remaining are missing the 'threat:triage:score' and should be sorted last
- Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(8).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(9).getSource().containsKey("threat:triage:score"));
- }
- //Sort ascending with missing fields
- {
- SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, results.size());
-
- // the remaining are missing the 'threat:triage:score' and should be sorted last
- Assert.assertFalse(results.get(0).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(1).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score"));
-
- // validate sorted order - there are only 2 with a 'threat:triage:score'
- Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
- Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
- }
- //pagination test case
- {
- SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(3, results.size());
- Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
- Assert.assertEquals(6, results.get(0).getSource().get("timestamp"));
- Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
- Assert.assertEquals(5, results.get(1).getSource().get("timestamp"));
- Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
- Assert.assertEquals(4, results.get(2).getSource().get("timestamp"));
- }
- //Index query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(5, response.getTotal());
- List<SearchResult> results = response.getResults();
- for(int i = 5,j=0;i > 0;i--,j++) {
- Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
- Assert.assertEquals(i, results.get(j).getSource().get("timestamp"));
- }
- }
- //Facet query including all field types
- {
- SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(12, response.getTotal());
-
- Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
- Assert.assertEquals(8, facetCounts.size());
-
- // source:type
- Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
- Assert.assertEquals(2, sourceTypeCounts.size());
- Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
- Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
-
- // ip_src_addr
- Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr");
- Assert.assertEquals(8, ipSrcAddrCounts.size());
- Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8"));
-
- // ip_src_port
- Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port");
- Assert.assertEquals(10, ipSrcPortCounts.size());
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010"));
-
- // long_field
- Map<String, Long> longFieldCounts = facetCounts.get("long_field");
- Assert.assertEquals(2, longFieldCounts.size());
- Assert.assertEquals(new Long(8), longFieldCounts.get("10000"));
- Assert.assertEquals(new Long(2), longFieldCounts.get("20000"));
-
- // timestamp
- Map<String, Long> timestampCounts = facetCounts.get("timestamp");
- Assert.assertEquals(10, timestampCounts.size());
- Assert.assertEquals(new Long(1), timestampCounts.get("1"));
- Assert.assertEquals(new Long(1), timestampCounts.get("2"));
- Assert.assertEquals(new Long(1), timestampCounts.get("3"));
- Assert.assertEquals(new Long(1), timestampCounts.get("4"));
- Assert.assertEquals(new Long(1), timestampCounts.get("5"));
- Assert.assertEquals(new Long(1), timestampCounts.get("6"));
- Assert.assertEquals(new Long(1), timestampCounts.get("7"));
- Assert.assertEquals(new Long(1), timestampCounts.get("8"));
- Assert.assertEquals(new Long(1), timestampCounts.get("9"));
- Assert.assertEquals(new Long(1), timestampCounts.get("10"));
-
- // latitude
- Map<String, Long> latitudeCounts = facetCounts.get("latitude");
- Assert.assertEquals(2, latitudeCounts.size());
- List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet());
- Collections.sort(latitudeKeys);
- Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001);
- Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
- Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
- Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
-
- // score
- Map<String, Long> scoreFieldCounts = facetCounts.get("score");
- Assert.assertEquals(4, scoreFieldCounts.size());
- List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
- Collections.sort(scoreFieldKeys);
- Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
- Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
- Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
- Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
- Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
- Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
- Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
- Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
-
- // is_alert
- Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
- Assert.assertEquals(2, isAlertCounts.size());
- Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
- Assert.assertEquals(new Long(4), isAlertCounts.get("false"));
- }
- //Bad facet query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class);
- try {
- dao.search(request);
- Assert.fail("Exception expected, but did not come.");
- }
- catch(InvalidSearchException ise) {
- // success
- }
- }
- //Disabled facet query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertNull(response.getFacetCounts());
+ public void get_all_latest_guid() throws Exception {
+ List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>() {
+ });
+ Map<String, Document> docs = new HashMap<>();
+
+ for(Document doc : dao.getAllLatest(request)) {
+ docs.put(doc.getGuid(), doc);
}
+ Assert.assertEquals(2, docs.size());
+ Assert.assertTrue(docs.keySet().contains("bro_1"));
+ Assert.assertTrue(docs.keySet().contains("snort_2"));
+ Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type"));
+ Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type"));
}
@Test
@@ -725,6 +504,42 @@ public abstract class SearchIntegrationTest {
}
@Test
+ public void sort_ascending_with_missing_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, results.size());
+
+ // the remaining are missing the 'threat:triage:score' and should be sorted last
+ for (int i = 0; i < 8; i++) {
+ Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
+ }
+
+ // validate sorted order - there are only 2 with a 'threat:triage:score'
+ Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
+ Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
+ }
+
+ @Test
+ public void sort_descending_with_missing_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, results.size());
+
+ // validate sorted order - there are only 2 with a 'threat:triage:score'
+ Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
+ Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
+
+ // the remaining are missing the 'threat:triage:score' and should be sorted last
+ for (int i = 2; i < 10; i++) {
+ Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
+ }
+ }
+
+ @Test
public void results_are_paginated() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
SearchResponse response = dao.search(request);
@@ -755,7 +570,7 @@ public abstract class SearchIntegrationTest {
public void facet_query_yields_field_types() throws Exception {
SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
SearchResponse response = dao.search(request);
- Assert.assertEquals(12, response.getTotal());
+ Assert.assertEquals(10, response.getTotal());
Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
Assert.assertEquals(8, facetCounts.size());
Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
@@ -1110,15 +925,15 @@ public abstract class SearchIntegrationTest {
}
@Test
- public void searches_metaalerts_fields() throws Exception {
- SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class);
+ public void sort_by_guid() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
SearchResponse response = dao.search(request);
- Assert.assertEquals(2, response.getTotal());
+ Assert.assertEquals(5, response.getTotal());
List<SearchResult> results = response.getResults();
- for (int i = 0; i < 2; ++i) {
+ for (int i = 0; i < 5; ++i) {
Map<String, Object> source = results.get(i).getSource();
Assert.assertEquals(1, source.size());
- Assert.assertEquals(source.get("guid"), "meta_" + (i + 1));
+ Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
}
}
[13/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/files/bro.out
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/files/bro.out b/metron-deployment/roles/sensor-stubs/files/bro.out
deleted file mode 100644
index 09cacfd..0000000
--- a/metron-deployment/roles/sensor-stubs/files/bro.out
+++ /dev/null
@@ -1,1346 +0,0 @@
-{"dns": {"ts":1484167797.685113,"uid":"C6MvbX3LOBTaZxwFCd","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484167798.055447,"uid":"CgA8NA4MlyRJmknKpf","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F7E6kW3Y9gaoHrgMCf"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167798.772618,"uid":"Cf322m3qSDKAsR8hTi","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Ftwsgz49riOg3FHK8l"],"resp_mime_types":["application/x-shockwave-flash"]}}
-{"dns": {"ts":1484167798.493574,"uid":"CZhu2h4SgVZWdW1aQl","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"dns": {"ts":1484167798.494185,"uid":"Czlibt2NgLkfLp7FPh","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484167799.934294,"uid":"CsWciw2WfERVM0Aczg","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fvq86i4oLSojHBFLSj"]}}
-{"http": {"ts":1484167800.359204,"uid":"C4airn3x7y3KVhZy8i","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJMtKEAMFuMvzDFJ7"],"resp_mime_types":["application/x-dosexec"]}}
-{"http": {"ts":1484167802.758608,"uid":"CJFa5h4HKSqVViZwbb","id.orig_h":"192.168.138.158","id.orig_p":49190,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b2566564b3ba1a38e61c83957a7dbcd5","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167803.072425,"uid":"CMOdJA4OzlvgzCK2qf","id.orig_h":"192.168.138.158","id.orig_p":49191,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?3a08b0be8322c244f5a1cb9c1057d941","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167803.402485,"uid":"CJVA893e60mcz43Jrj","id.orig_h":"192.168.138.158","id.orig_p":49192,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?d71e0bd86db9587158745a986a4b3606","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167804.009067,"uid":"C1rmUO2zDsIbgBR8Ik","id.orig_h":"192.168.138.158","id.orig_p":49193,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?34eaf8bd50d85d8c6baacb45f0a7b22e","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167804.312623,"uid":"CCTaln3ggV4dOqGETi","id.orig_h":"192.168.138.158","id.orig_p":49194,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?60dbe33b908e0086292196ef001816bc","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167804.557307,"uid":"CK6DaM2pfjgwi3pY8j","id.orig_h":"192.168.138.158","id.orig_p":49197,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?t=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["Fkypgg28hizg2EwJRk"],"orig_mime_types":["text/plain"],"resp_fuids":["FqQOjx3rFxTb4RSHE9"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167804.624712,"uid":"Clghht2drNjg3G5dPh","id.orig_h":"192.168.138.158","id.orig_p":49196,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?51424ddd486ff06861fceed24e86b329","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"dns": {"ts":1484167804.053752,"uid":"CoiTkw2sb9stNr10zg","id.orig_h":"192.168.138.158","id.orig_p":53571,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":15553,"query":"ip-addr.es","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["188.165.164.184"],"TTLs":[21599.0],"rejected":false}}
-{"dns": {"ts":1484167804.472938,"uid":"CJodZl3aVCrbHCw7xk","id.orig_h":"192.168.138.158","id.orig_p":61720,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":23625,"query":"runlove.us","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["204.152.254.221"],"TTLs":[14069.0],"rejected":false}}
-{"dns": {"ts":1484167804.737519,"uid":"CkV7Z23iTRHYS1MxCh","id.orig_h":"192.168.138.158","id.orig_p":50509,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":6088,"query":"kritischerkonsum.uni-koeln.de","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484167804.961078,"uid":"CBwIqk3VHRfD9CapGl","id.orig_h":"192.168.138.158","id.orig_p":56753,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":41589,"query":"comarksecurity.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["72.34.49.86"],"TTLs":[13888.0],"rejected":false}}
-{"http": {"ts":1484167805.046633,"uid":"CLfH0q2XJiBH0gUngj","id.orig_h":"192.168.138.158","id.orig_p":49198,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?c=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["F5sGcb3aKKVdqlR4l8"],"orig_mime_types":["text/plain"],"resp_fuids":["FfjQv01KPJAoEcMH1b"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167807.20478,"uid":"CgI9Lp32cTchxqp8Wk","id.orig_h":"192.168.138.158","id.orig_p":49199,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?l=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FDpZNy3tiCh1cjvs19"],"orig_mime_types":["text/plain"],"resp_fuids":["FCCDfF1umBiOBkbAl3"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167807.449353,"uid":"CuowPb45wYWpb50JTe","id.orig_h":"192.168.138.158","id.orig_p":49200,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?t=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":996,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FPM0BMoMv2KB8kbg6"],"orig_mime_types":["text/plain"],"resp_fuids":["F8YBeS1viVGE8sJCOe"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167810.18734,"uid":"ClAWUw4JlQ6WlHFWCc","id.orig_h":"192.168.138.158","id.orig_p":49201,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FMQqcF3hxESuoUxoAi"],"orig_mime_types":["text/plain"],"resp_fuids":["FGxTQj2tNlpA4iyAM7"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167810.407854,"uid":"CTC5QZ37i6qEO6MHef","id.orig_h":"192.168.138.158","id.orig_p":49202,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":45662,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FbWILCRYdYGyU9ZRk"],"orig_mime_types":["text/plain"],"resp_fuids":["FS225I2iUF28vOdi7i"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167816.579111,"uid":"CzJzUp2mgm3oMfTpxk","id.orig_h":"192.168.138.158","id.orig_p":49203,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?f=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["F7wKbu2LQZ2q8jSfc5"],"orig_mime_types":["text/plain"],"resp_fuids":["FF9gh04PL6B9Cvlbbl"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167816.846125,"uid":"CozvfM21cOxjQJvB8j","id.orig_h":"192.168.138.158","id.orig_p":49204,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FjWkDS5TCYn9GYRCd"],"orig_mime_types":["text/plain"],"resp_fuids":["FgMrsN3zfIRJ9ipWq6"],"resp_mime_types":["text/plain"]}}
-{"dns": {"ts":1484167810.995898,"uid":"CC18jH3AC7y8NPST2b","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"http": {"ts":1484167804.326254,"uid":"CwFhR03V5JXOYF8Ppk","id.orig_h":"192.168.138.158","id.orig_p":49195,"id.resp_h":"188.165.164.184","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ip-addr.es","uri":"/","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"tags":[]}}
-{"dns": {"ts":1484167842.890717,"uid":"CTvVE015EfMtdMxkU7","id.orig_h":"192.168.138.158","id.orig_p":50329,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":5810,"query":"7oqnsnzwwnm6zb7y.gigapaysun.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["95.163.121.204"],"TTLs":[14277.0],"rejected":false}}
-{"http": {"ts":1484167843.102031,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":3289,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fmv6Ap2EAcThJKped6"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167845.759905,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":4492,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FPQd6A1L85pPPCM7ya"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167847.5693,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/it.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":552,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fitq9q43ZupDyYoAyk"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167847.568708,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/us.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":825,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FSuqni2XAHJRnN8bYh"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167847.760685,"uid":"Cg6qof2daZW7072Gq4","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":240,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FTT0j8hWM1ENBWINg"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167847.759652,"uid":"CemTNB1OFxbrBn2wD2","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/picture.php?k=11iqmfg\u0026b7f2a994c3eaaf014608b272c46cf764","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":1823,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FKu4gy3zUcygg980ee"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167847.760176,"uid":"Cdg2Cf1BnvStDcNm44","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/es.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":634,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FoY2NUzH7asqxomge"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167847.776105,"uid":"CUrJ3S149MGwkEQcc","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/de.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":534,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fv44d82Oy08X20w3jh"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167849.341698,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/fr.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":694,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F7znd92YPrDeWJkLWb"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167849.558631,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":242,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FKJ5Y92fddrzBWKCb8"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167849.59428,"uid":"Cg6qof2daZW7072Gq4","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":239,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F3R4B2XMTLO3hSure"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167849.692474,"uid":"CemTNB1OFxbrBn2wD2","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":237,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FOoUNQ3OUe0r3e9Ewa"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167851.825173,"uid":"Cdg2Cf1BnvStDcNm44","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/favicon.ico","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":318,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F0ASzM1opxGAKE6oMe"],"resp_mime_types":["image/x-icon"]}}
-{"http": {"ts":1484167856.767294,"uid":"CUrJ3S149MGwkEQcc","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"POST","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":14,"response_body_len":14641,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FcUgiOBXtoqaQLmed"],"orig_mime_types":["text/plain"],"resp_fuids":["Fp1E561lGpI5pr3S8e"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167859.506315,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":304,"status_msg":"Not Modified","tags":[]}}
-{"http": {"ts":1484167861.613787,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/bitcoin.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":5523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FZv62C25nyrRv26Mhl"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167861.614577,"uid":"Cg6qof2daZW7072Gq4","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/button_pay.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":727,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FICI2E4Vfpq8wOCTGc"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167798.787714,"uid":"CK8vN03Rc0FuI0R6qk","id.orig_h":"192.168.138.158","id.orig_p":49186,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":121635,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F4hDsd3d3xB7Rhef5i"],"resp_mime_types":["text/html"]}}
-{"dns": {"ts":1484167890.789624,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484167891.791607,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484167894.797258,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484167903.814905,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"http": {"ts":1484167923.810197,"uid":"CaSyqTwPFwYZLa4ra","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FmsYkt2F76v7XUBNAd"],"resp_mime_types":["text/html"]}}
-{"dns": {"ts":1484167923.441012,"uid":"C21R6O7H0Kzv5Fdd6","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484167924.511788,"uid":"CRGLdEasAJUDL8Tu4","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FHMpUl2B1lUkpzZoQi"],"resp_mime_types":["application/x-shockwave-flash"]}}
-{"dns": {"ts":1484167924.233822,"uid":"CwNs81Nmjja4ubh79","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"dns": {"ts":1484167924.234067,"uid":"C5GlhlwTDTSQ0b7T7","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484167925.62215,"uid":"Cprvbi2qWL0ZpQ3hT1","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FiZAai2ofPvEPp5aC8"]}}
-{"http": {"ts":1484167926.026722,"uid":"CWFPu12Z6Poy9LL6q3","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FMtkgchK5p70TpU2"],"resp_mime_types":["application/x-dosexec"]}}
-{"http": {"ts":1484167928.239914,"uid":"CuRsfK1BjhmhzlDV45","id.orig_h":"192.168.138.158","id.orig_p":49190,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b2566564b3ba1a38e61c83957a7dbcd5","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167928.552572,"uid":"CuY6Oe1cwQi28U2sW","id.orig_h":"192.168.138.158","id.orig_p":49191,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?3a08b0be8322c244f5a1cb9c1057d941","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167928.866681,"uid":"C09cX52HiXhkT5Exa2","id.orig_h":"192.168.138.158","id.orig_p":49192,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?d71e0bd86db9587158745a986a4b3606","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167929.401205,"uid":"Cb5L1N1Ug32x6Oz4u3","id.orig_h":"192.168.138.158","id.orig_p":49193,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?34eaf8bd50d85d8c6baacb45f0a7b22e","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167929.705419,"uid":"CAhsP32ytUNibnJDX4","id.orig_h":"192.168.138.158","id.orig_p":49194,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?60dbe33b908e0086292196ef001816bc","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484167929.948512,"uid":"Cj4sal27hxyUSBNdG4","id.orig_h":"192.168.138.158","id.orig_p":49197,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?t=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FHBLd84QtlbCemYez2"],"orig_mime_types":["text/plain"],"resp_fuids":["FaaROi4lrbjm1FIpBa"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167930.004132,"uid":"CZiWcw1wYkyuuJ7dn2","id.orig_h":"192.168.138.158","id.orig_p":49196,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?51424ddd486ff06861fceed24e86b329","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"dns": {"ts":1484167929.445849,"uid":"ChakWs1TGdxYbia8H","id.orig_h":"192.168.138.158","id.orig_p":53571,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":15553,"query":"ip-addr.es","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["188.165.164.184"],"TTLs":[21599.0],"rejected":false}}
-{"dns": {"ts":1484167929.865123,"uid":"CVy9AO1a2S9g1ME1D","id.orig_h":"192.168.138.158","id.orig_p":61720,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":23625,"query":"runlove.us","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["204.152.254.221"],"TTLs":[14069.0],"rejected":false}}
-{"dns": {"ts":1484167930.106812,"uid":"CFdDBq1CQI2ScUH7c3","id.orig_h":"192.168.138.158","id.orig_p":50509,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":6088,"query":"kritischerkonsum.uni-koeln.de","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484167930.293382,"uid":"Co0gpI15MTwRMCkjN1","id.orig_h":"192.168.138.158","id.orig_p":56753,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":41589,"query":"comarksecurity.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["72.34.49.86"],"TTLs":[13888.0],"rejected":false}}
-{"http": {"ts":1484167930.378501,"uid":"CGfX4Z1TFOkZNVV1Ba","id.orig_h":"192.168.138.158","id.orig_p":49198,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?c=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FO2LTy2hFvukwN9GFi"],"orig_mime_types":["text/plain"],"resp_fuids":["FacbaL1rZYmMXVxSC7"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167932.531381,"uid":"CayZYU7BSUC868ND7","id.orig_h":"192.168.138.158","id.orig_p":49199,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?l=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["F8A2tO2f0GW4SwIiQ1"],"orig_mime_types":["text/plain"],"resp_fuids":["FXQqrf3jpgyu4s2Rka"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167932.764152,"uid":"CrLZ5CuSEtlRc7ZZ8","id.orig_h":"192.168.138.158","id.orig_p":49200,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?t=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":996,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["F8IiLaaCLRBk820Zh"],"orig_mime_types":["text/plain"],"resp_fuids":["FDNU5S22CAykjNpx9b"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167935.492514,"uid":"CvFLziqvgf8GKC8Y9","id.orig_h":"192.168.138.158","id.orig_p":49201,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FtWE1u3rLNc2AU2ZLh"],"orig_mime_types":["text/plain"],"resp_fuids":["FtIwWe2cQZ51hQ8Rv5"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167935.713955,"uid":"CLq0pehvmEuCqLi06","id.orig_h":"192.168.138.158","id.orig_p":49202,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":45662,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FB2jupcXqhiMRiem6"],"orig_mime_types":["text/plain"],"resp_fuids":["FVDw1t7CSEasxpe33"],"resp_mime_types":["image/png"]}}
-{"dns": {"ts":1484167930.866489,"uid":"Cu87p4iMM0QdpiTp6","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"http": {"ts":1484167941.875595,"uid":"CTWWO71KlU9DRrmnp7","id.orig_h":"192.168.138.158","id.orig_p":49203,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?f=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FpJNmm2r9zrMR9EgVg"],"orig_mime_types":["text/plain"],"resp_fuids":["FRwLTg9W436fghxr5"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167942.133849,"uid":"CcQP5R1PG4q2f1yT8","id.orig_h":"192.168.138.158","id.orig_p":49204,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FOy5gi3M2w8XWvNYqc"],"orig_mime_types":["text/plain"],"resp_fuids":["F4zMeEvy9RIBYHdt3"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167929.718969,"uid":"CU1z6c1RbgzgRhb2E3","id.orig_h":"192.168.138.158","id.orig_p":49195,"id.resp_h":"188.165.164.184","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ip-addr.es","uri":"/","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"tags":[]}}
-{"dns": {"ts":1484167968.175518,"uid":"CogghzajDK9MdvUb3","id.orig_h":"192.168.138.158","id.orig_p":50329,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":5810,"query":"7oqnsnzwwnm6zb7y.gigapaysun.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["95.163.121.204"],"TTLs":[14277.0],"rejected":false}}
-{"http": {"ts":1484167968.381714,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":3289,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FkXWFH1GnHj99Fx299"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167971.040006,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":4492,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJCJeM1R4x1H4y42dg"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484167972.844911,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/it.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":552,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fvu2eI2KDzXQvkD2Dh"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167972.844675,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/us.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":825,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FuSbCP13SdEpnVUFn"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167973.033914,"uid":"Ch2hGO1LrumL0QNPG","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":240,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FfcAU83ia8gfjLsWOk"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167973.033427,"uid":"CHwEGq1paXl7IqvzD3","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/picture.php?k=11iqmfg\u0026b7f2a994c3eaaf014608b272c46cf764","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":1823,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FfBfWC3nUGssCHBsL2"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167973.033645,"uid":"CzOF6l2DOLLYUZwvG4","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/es.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":634,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fil6QO3GWNnrBAiUf"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167973.049027,"uid":"Cbhgaw1IVL6NGqHpn2","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/de.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":534,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F4cZLM1Rfj48wYg1Pb"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167974.613828,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/fr.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":694,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FpjJ2mpIuKnU39Gve"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167974.82954,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":242,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F9K2Bp4ET8NJaBib48"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167974.864613,"uid":"Ch2hGO1LrumL0QNPG","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":239,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FY7LMa40QDG4zeNP5j"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167974.96311,"uid":"CHwEGq1paXl7IqvzD3","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":237,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FmYeOPLgRfGvDw3hj"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167977.095491,"uid":"CzOF6l2DOLLYUZwvG4","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/favicon.ico","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":318,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FUi3FI31tCwo4WgXS3"],"resp_mime_types":["image/x-icon"]}}
-{"http": {"ts":1484167982.036116,"uid":"Cbhgaw1IVL6NGqHpn2","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"POST","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":14,"response_body_len":14641,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FC94vR3f72342iAX92"],"orig_mime_types":["text/plain"],"resp_fuids":["Fk4Uzu2jQqXKUB00d5"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484167984.773279,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":304,"status_msg":"Not Modified","tags":[]}}
-{"http": {"ts":1484167986.879607,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/bitcoin.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":5523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FPcM0x2VeblSWANUz6"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484167986.880042,"uid":"Ch2hGO1LrumL0QNPG","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/button_pay.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":727,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fvv1TH12c7JL4UQUa2"],"resp_mime_types":["image/png"]}}
-{"dns": {"ts":1484167995.474042,"uid":"CgRzK32SqF5wJvC0Y4","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"http": {"ts":1484167924.526122,"uid":"CLKLkp1z9ZWAE0eou","id.orig_h":"192.168.138.158","id.orig_p":49186,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":121635,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FrcnSsZqVzpjB9o3j"],"resp_mime_types":["text/html"]}}
-{"dns": {"ts":1484168010.822008,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168011.828379,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168014.836836,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168023.85904,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168048.700291,"uid":"Co6RaRZQIXvwkvaT8","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484168049.069438,"uid":"CeVtdzaICCMxZFAY9","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fx1AH04QmVyAYM5kd9"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168049.769399,"uid":"CZOU9CQKfQzbTKGZ8","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F95sxB3DPck4oMGLmc"],"resp_mime_types":["application/x-shockwave-flash"]}}
-{"dns": {"ts":1484168049.492644,"uid":"CXiPrelEswy2Vy506","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"dns": {"ts":1484168049.492947,"uid":"CddYI711hltuYI1aE7","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484168050.884154,"uid":"CyAev4UQJHk5ECqp6","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FxyuCg3ehaL3Q3Jicf"]}}
-{"http": {"ts":1484168051.288742,"uid":"CpF3KK32282sEULktb","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FbsI7rQSYdyoN8osc"],"resp_mime_types":["application/x-dosexec"]}}
-{"http": {"ts":1484168053.472796,"uid":"CTUxsD4ZOi3CcOT5Tc","id.orig_h":"192.168.138.158","id.orig_p":49190,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b2566564b3ba1a38e61c83957a7dbcd5","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484168053.7841,"uid":"CYsXRm4a452wUqMdpe","id.orig_h":"192.168.138.158","id.orig_p":49191,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?3a08b0be8322c244f5a1cb9c1057d941","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484168054.099112,"uid":"CQclu54qLZ704kkF4g","id.orig_h":"192.168.138.158","id.orig_p":49192,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?d71e0bd86db9587158745a986a4b3606","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484168054.633374,"uid":"CCOkcA3TQkLzkoUtVb","id.orig_h":"192.168.138.158","id.orig_p":49193,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?34eaf8bd50d85d8c6baacb45f0a7b22e","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"http": {"ts":1484168054.936005,"uid":"CXj9s84H83bRMeXVse","id.orig_h":"192.168.138.158","id.orig_p":49194,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?60dbe33b908e0086292196ef001816bc","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"dns": {"ts":1484168054.677804,"uid":"C0zadr4MkQXXg3R6ad","id.orig_h":"192.168.138.158","id.orig_p":53571,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":15553,"query":"ip-addr.es","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["188.165.164.184"],"TTLs":[21599.0],"rejected":false}}
-{"dns": {"ts":1484168055.095134,"uid":"ChdTCB23ylNPAWtcHj","id.orig_h":"192.168.138.158","id.orig_p":61720,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":23625,"query":"runlove.us","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["204.152.254.221"],"TTLs":[14069.0],"rejected":false}}
-{"http": {"ts":1484168055.178325,"uid":"C48TgT2oKyquocJlgi","id.orig_h":"192.168.138.158","id.orig_p":49197,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?t=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["Fn5ziO3r1iNx37j0Ml"],"orig_mime_types":["text/plain"],"resp_fuids":["Fqi2yO1cKqX6xLdMEe"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168055.233385,"uid":"C7A9tv3exoi6fTWTbl","id.orig_h":"192.168.138.158","id.orig_p":49196,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?51424ddd486ff06861fceed24e86b329","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
-{"dns": {"ts":1484168055.336209,"uid":"CQgJw93u2weQeXL7ch","id.orig_h":"192.168.138.158","id.orig_p":50509,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":6088,"query":"kritischerkonsum.uni-koeln.de","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168055.522671,"uid":"CcLbIe3Hv8tmqEXAql","id.orig_h":"192.168.138.158","id.orig_p":56753,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":41589,"query":"comarksecurity.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["72.34.49.86"],"TTLs":[13888.0],"rejected":false}}
-{"http": {"ts":1484168055.607448,"uid":"CUlrzm2SQPBB5osl6k","id.orig_h":"192.168.138.158","id.orig_p":49198,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?c=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["F4QJql3CmvlqjPZ62c"],"orig_mime_types":["text/plain"],"resp_fuids":["FvG9Y93qeCBMqZaxl2"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484168057.760379,"uid":"CdUJwG2Df90m0Y7OSi","id.orig_h":"192.168.138.158","id.orig_p":49199,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?l=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["Fh9CoH303MQ3vTRjB"],"orig_mime_types":["text/plain"],"resp_fuids":["F9iisA25ZMf02F0vS5"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168057.992988,"uid":"CRAvCZ2ozjOE5ZgU9d","id.orig_h":"192.168.138.158","id.orig_p":49200,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?t=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":996,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FslJp33fnKpZkHE8y2"],"orig_mime_types":["text/plain"],"resp_fuids":["FJ8xAA0UPebOG5gAc"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484168060.720375,"uid":"CXT1m84PFOVKcQCote","id.orig_h":"192.168.138.158","id.orig_p":49201,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FOsK283Q71ZIaQFli"],"orig_mime_types":["text/plain"],"resp_fuids":["FqXoIQ1Wo0FMhwXkm"],"resp_mime_types":["text/html"]}}
-{"dns": {"ts":1484168050.914926,"uid":"CopjWUPbl0jgVvxuf","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"http": {"ts":1484168060.941201,"uid":"C759hO3qWwA0XQqi4g","id.orig_h":"192.168.138.158","id.orig_p":49202,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":45662,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FruxXL3ovqxbl9ZEt1"],"orig_mime_types":["text/plain"],"resp_fuids":["FXITBL1jZa8vBZWhOd"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168067.100666,"uid":"C8Np8A35YX2amygMVb","id.orig_h":"192.168.138.158","id.orig_p":49203,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?f=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FYxhwp2IpRU8tG2ED1"],"orig_mime_types":["text/plain"],"resp_fuids":["F7VCjtX86bUyVy3R2"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168067.360726,"uid":"CI33ir4V9AqRYIQDSc","id.orig_h":"192.168.138.158","id.orig_p":49204,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FSMLvV1wZHnHEWiRg6"],"orig_mime_types":["text/plain"],"resp_fuids":["FbdBIa2cOLEETy8ZM6"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484168054.949144,"uid":"CpYseO3TyKre605q5h","id.orig_h":"192.168.138.158","id.orig_p":49195,"id.resp_h":"188.165.164.184","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ip-addr.es","uri":"/","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"tags":[]}}
-{"dns": {"ts":1484168093.401627,"uid":"CQC8Pm4Qo5KRLMYqpe","id.orig_h":"192.168.138.158","id.orig_p":50329,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":5810,"query":"7oqnsnzwwnm6zb7y.gigapaysun.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["95.163.121.204"],"TTLs":[14277.0],"rejected":false}}
-{"http": {"ts":1484168093.608251,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":3289,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FOov1rV6rL28n8qy1"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168096.264793,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":4492,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FZjJMI3o09BdFRIgU7"],"resp_mime_types":["text/plain"]}}
-{"http": {"ts":1484168098.07042,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/it.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":552,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FEoclLaLEjZvuZZt9"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168098.069604,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/us.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":825,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FelBi52nX055gNTqoh"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168098.259569,"uid":"CuQjtm2b1ZTs5e0AHj","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":240,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F39o293n7WwhocflC7"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168098.259081,"uid":"CrtAyZ2wczgi6YrAch","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/picture.php?k=11iqmfg\u0026b7f2a994c3eaaf014608b272c46cf764","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":1823,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FB79Dm1t9dSUOI5eY6"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168098.259316,"uid":"CBmCBG2XG9D8KFerSi","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/es.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":634,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJ14zE3SRYHfEVUBK6"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168098.274642,"uid":"C4PRKe3tPbLrANnoql","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/de.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":534,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fh4WuDHfRrX302wDj"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168099.839755,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/fr.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":694,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F8viU44xUigTtf4F4"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168100.055845,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":242,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FlQf7j3HXl7ZvoKcwe"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168100.091007,"uid":"CuQjtm2b1ZTs5e0AHj","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":239,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FqRNLE44JylF8wuffj"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168100.188585,"uid":"CrtAyZ2wczgi6YrAch","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":237,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FAq3rD1EezTPYh1XVi"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168102.320148,"uid":"CBmCBG2XG9D8KFerSi","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/favicon.ico","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":318,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FsRCRQ1rW7SviqZ7rf"],"resp_mime_types":["image/x-icon"]}}
-{"dns": {"ts":1484168094.52272,"uid":"Chr2FK36PUE9wFhItb","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"http": {"ts":1484168107.260553,"uid":"C4PRKe3tPbLrANnoql","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"POST","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":14,"response_body_len":14641,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FoLJFr1bniaMScBzXa"],"orig_mime_types":["text/plain"],"resp_fuids":["FMKGGw1xJPXlLO2r2"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168109.998211,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":304,"status_msg":"Not Modified","tags":[]}}
-{"http": {"ts":1484168112.105378,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/bitcoin.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":5523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FGcm94EWzm8st4LQj"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168112.106225,"uid":"CuQjtm2b1ZTs5e0AHj","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/button_pay.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":727,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FpDS0Lu22MwpGf8ac"],"resp_mime_types":["image/png"]}}
-{"http": {"ts":1484168049.78326,"uid":"CJ8TuiTOmndD9weBa","id.orig_h":"192.168.138.158","id.orig_p":49186,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":121635,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FXdaxF1PjrvmYbJfLg"],"resp_mime_types":["text/html"]}}
-{"dns": {"ts":1484168130.768775,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168131.774252,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168134.783524,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168143.802793,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
-{"dns": {"ts":1484168173.926757,"uid":"CVf8zv3sBOdNwWTrbl","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"dns": {"ts":1484168174.717258,"uid":"CCbfqc3ox2hz3kqRSj","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"dns": {"ts":1484168174.717487,"uid":"CSHdCU2z9CrpPiMuXk","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
-{"http": {"ts":1484168174.29515,"uid":"CHg4AB2DzmEvPnlJoi","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FbYOC42kRE93hxmUOb"],"resp_mime_types":["text/html"]}}
-{"http": {"ts":1484168174.994035,"uid":"C3MrJz2uc4HxL7lCZg","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fz99Ml4Hgbcn29QSMa"],"resp_mime_types":["application/x-shockwave-flash"]}}
-{"http": {"ts":1484168176.096419,"uid":"CZerqX2fu5NcfhGiZj","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fixn8N2eNk8O3WwOti"]}}
-{"http": {"ts":1484168176.500568,"uid":"C7fLUD28ahyzAuVnAl","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F
<TRUNCATED>
[03/50] [abbrv] metron git commit: METRON-1426:
SensorIndexingConfigControllerIntegrationTest fails intermittently closes
apache/metron#906
Posted by rm...@apache.org.
METRON-1426: SensorIndexingConfigControllerIntegrationTest fails intermittently closes apache/metron#906
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/acb8b928
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/acb8b928
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/acb8b928
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: acb8b9287e8b6a2a0ced60855f0e5d6e6c343062
Parents: 40411d4
Author: mmiklavc <mi...@gmail.com>
Authored: Thu Jan 25 09:16:01 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Jan 25 09:16:01 2018 -0500
----------------------------------------------------------------------
...IndexingConfigControllerIntegrationTest.java | 54 +++++++++++---------
1 file changed, 30 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/acb8b928/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
index 28977fd..d78bedf 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/controller/SensorIndexingConfigControllerIntegrationTest.java
@@ -17,6 +17,18 @@
*/
package org.apache.metron.rest.controller;
+import static org.apache.metron.integration.utils.TestUtils.assertEventually;
+import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
+import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
+import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
+import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
+import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
+
import org.adrianwalker.multilinestring.Multiline;
import org.apache.metron.rest.service.SensorIndexingConfigService;
import org.junit.Before;
@@ -31,18 +43,6 @@ import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.web.context.WebApplicationContext;
-import static org.apache.metron.integration.utils.TestUtils.assertEventually;
-import static org.apache.metron.rest.MetronRestConstants.TEST_PROFILE;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.csrf;
-import static org.springframework.security.test.web.servlet.request.SecurityMockMvcRequestPostProcessors.httpBasic;
-import static org.springframework.security.test.web.servlet.setup.SecurityMockMvcConfigurers.springSecurity;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
-import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.content;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
-import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
-
@RunWith(SpringRunner.class)
@SpringBootTest(webEnvironment= SpringBootTest.WebEnvironment.RANDOM_PORT)
@ActiveProfiles(TEST_PROFILE)
@@ -79,14 +79,17 @@ public class SensorIndexingConfigControllerIntegrationTest {
this.mockMvc.perform(post(sensorIndexingConfigUrl).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
.andExpect(status().isUnauthorized());
- this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest"))
- .andExpect(status().isUnauthorized());
+ assertEventually(() -> this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest"))
+ .andExpect(status().isUnauthorized())
+ );
- this.mockMvc.perform(get(sensorIndexingConfigUrl))
- .andExpect(status().isUnauthorized());
+ assertEventually(() -> this.mockMvc.perform(get(sensorIndexingConfigUrl))
+ .andExpect(status().isUnauthorized())
+ );
- this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(csrf()))
- .andExpect(status().isUnauthorized());
+ assertEventually(() -> this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(csrf()))
+ .andExpect(status().isUnauthorized())
+ );
}
@Test
@@ -114,10 +117,11 @@ public class SensorIndexingConfigControllerIntegrationTest {
.andExpect(jsonPath("$.index").value("broTest"))
.andExpect(jsonPath("$.batchSize").value(1));
- this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
+ assertEventually(() -> this.mockMvc.perform(get(sensorIndexingConfigUrl + "/list/indices/elasticsearch").with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(content().bytes("[\"broTest\"]".getBytes()));
+ .andExpect(content().bytes("[\"broTest\"]".getBytes()))
+ );
assertEventually(() -> this.mockMvc.perform(post(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user, password)).with(csrf()).contentType(MediaType.parseMediaType("application/json;charset=UTF-8")).content(broJson))
.andExpect(status().isOk())
@@ -142,16 +146,18 @@ public class SensorIndexingConfigControllerIntegrationTest {
this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
.andExpect(status().isOk());
- this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)))
- .andExpect(status().isNotFound());
+ assertEventually(() -> this.mockMvc.perform(get(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)))
+ .andExpect(status().isNotFound())
+ );
this.mockMvc.perform(delete(sensorIndexingConfigUrl + "/broTest").with(httpBasic(user,password)).with(csrf()))
.andExpect(status().isNotFound());
- this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password)))
+ assertEventually(() -> this.mockMvc.perform(get(sensorIndexingConfigUrl).with(httpBasic(user,password)))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.parseMediaType("application/json;charset=UTF-8")))
- .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist());
+ .andExpect(jsonPath("$[?(@.sensorTopic == 'broTest')]").doesNotExist())
+ );
sensorIndexingConfigService.delete("broTest");
}
[44/50] [abbrv] metron git commit: METRON-1451 On Centos full dev,
Metron Indexing shows up as stopped (anandsubbu via merrimanr) closes
apache/metron#932
Posted by rm...@apache.org.
METRON-1451 On Centos full dev, Metron Indexing shows up as stopped (anandsubbu via merrimanr) closes apache/metron#932
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fa86663e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fa86663e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fa86663e
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: fa86663ecdb166f889c9c0c4c34373aec68dbd30
Parents: 48d9d25
Author: anandsubbu <as...@HW12724.local>
Authored: Thu Feb 8 16:19:39 2018 -0600
Committer: merrimanr <me...@apache.org>
Committed: Thu Feb 8 16:19:39 2018 -0600
----------------------------------------------------------------------
.../ansible/roles/ambari_config/tasks/dependencies.yml | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fa86663e/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
index fdb21c7..52d6ca1 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
@@ -15,10 +15,7 @@
# limitations under the License.
#
---
-- name: Install python-requests on CentOS
- yum: name=python-requests
- when: ansible_distribution == "CentOS"
-
-- name: Install python-requests on Ubuntu
- apt: name=python-requests force=yes
- when: ansible_distribution == "Ubuntu"
+- name: Install python-requests module
+ pip:
+ name: requests
+ version: 2.6.1
[49/50] [abbrv] metron git commit: METRON-1394 Create Rest endpoint
to add the ACL for current user to kafka topics (MohanDV via merrimanr)
closes apache/metron#895
Posted by rm...@apache.org.
METRON-1394 Create Rest endpoint to add the ACL for current user to kafka topics (MohanDV via merrimanr) closes apache/metron#895
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e265b369
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e265b369
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e265b369
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: e265b369192613335335514bc842f6aebe016d3e
Parents: 5f08ba0
Author: MohanDV <mo...@gmail.com>
Authored: Wed Feb 21 08:32:01 2018 -0600
Committer: merrimanr <me...@apache.org>
Committed: Wed Feb 21 08:32:01 2018 -0600
----------------------------------------------------------------------
.../metron/rest/controller/KafkaController.java | 1 +
.../metron/rest/service/KafkaService.java | 9 ++++++
.../rest/service/impl/KafkaServiceImpl.java | 33 ++++++++++++++++++--
.../rest/service/impl/KafkaServiceImplTest.java | 10 ++++++
4 files changed, 51 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/e265b369/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
index d057ac4..d04e227 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/KafkaController.java
@@ -119,4 +119,5 @@ public class KafkaController {
kafkaService.produceMessage(name, message);
return new ResponseEntity<>(HttpStatus.OK);
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/e265b369/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
index da3b226..aa35c5d 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/KafkaService.java
@@ -68,4 +68,13 @@ public interface KafkaService {
String getSampleMessage(String topic);
void produceMessage(String topic, String message) throws RestException;
+
+
+ /**
+ *
+ * @param name The name of the Kafka topic to add the ACL.
+ * @return If topic was present true; otherwise false.
+ */
+ boolean addACLToCurrentUser(String name);
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/e265b369/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
index 4f232fb..ac001b5 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/KafkaServiceImpl.java
@@ -17,11 +17,13 @@
*/
package org.apache.metron.rest.service.impl;
-import java.util.HashSet;
+import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.Set;
+import java.util.HashSet;
+import java.util.Map;
import java.util.stream.Collectors;
+import kafka.admin.AclCommand;
import kafka.admin.AdminOperationException;
import kafka.admin.AdminUtils$;
import kafka.admin.RackAwareMode;
@@ -33,12 +35,15 @@ import org.apache.kafka.clients.producer.KafkaProducer;
import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
+import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.apache.metron.rest.model.KafkaTopic;
import org.apache.metron.rest.service.KafkaService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.env.Environment;
import org.springframework.kafka.core.ConsumerFactory;
+import org.springframework.security.core.context.SecurityContextHolder;
+import org.springframework.security.core.userdetails.User;
import org.springframework.stereotype.Service;
/**
@@ -83,6 +88,9 @@ public class KafkaServiceImpl implements KafkaService {
if (!listTopics().contains(topic.getName())) {
try {
adminUtils.createTopic(zkUtils, topic.getName(), topic.getNumPartitions(), topic.getReplicationFactor(), topic.getProperties(), RackAwareMode.Disabled$.MODULE$);
+ if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)){
+ addACLToCurrentUser(topic.getName());
+ }
} catch (AdminOperationException e) {
throw new RestException(e);
}
@@ -154,4 +162,25 @@ public class KafkaServiceImpl implements KafkaService {
public void produceMessage(String topic, String message) throws RestException {
kafkaProducer.send(new ProducerRecord<>(topic, message));
}
+
+ @Override
+ public boolean addACLToCurrentUser(String name){
+ if(listTopics().contains(name)) {
+ String zkServers = environment.getProperty(MetronRestConstants.ZK_URL_SPRING_PROPERTY);
+ User principal = (User) SecurityContextHolder.getContext().getAuthentication().getPrincipal();
+ String user = principal.getUsername();
+ List<String> cmd = new ArrayList<>();
+ cmd.add("--add");
+ cmd.add("--allow-principal");
+ cmd.add("User:" + user);
+ cmd.add("--topic");
+ cmd.add(name);
+ cmd.add("--authorizer-properties");
+ cmd.add("zookeeper.connect=" + String.join(",", zkServers));
+ AclCommand.main(cmd.toArray(new String[cmd.size()]));
+ } else {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/e265b369/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
index 4527e8e..b99128a 100644
--- a/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
+++ b/metron-interface/metron-rest/src/test/java/org/apache/metron/rest/service/impl/KafkaServiceImplTest.java
@@ -63,6 +63,7 @@ import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
import org.springframework.kafka.core.ConsumerFactory;
+
@SuppressWarnings("unchecked")
@RunWith(PowerMockRunner.class)
@PowerMockIgnore("javax.management.*") // resolve classloader conflict
@@ -79,6 +80,7 @@ public class KafkaServiceImplTest {
private KafkaService kafkaService;
+
private static final KafkaTopic VALID_KAFKA_TOPIC = new KafkaTopic() {{
setReplicationFactor(2);
setNumPartitions(1);
@@ -314,4 +316,12 @@ public class KafkaServiceImplTest {
verify(kafkaProducer).send(new ProducerRecord<>(topicName, expectedMessage));
verifyZeroInteractions(kafkaProducer);
}
+
+ @Test
+ public void addACLtoNonExistingTopicShouldReturnFalse() throws Exception{
+ when(kafkaConsumer.listTopics()).thenReturn(Maps.newHashMap());
+ assertFalse(kafkaService.addACLToCurrentUser("non_existent_topic"));
+ }
+
+
}
[05/50] [abbrv] metron git commit: METRON-1413 Add Metron Commit Tool
(nickwallen) closes apache/metron#902
Posted by rm...@apache.org.
METRON-1413 Add Metron Commit Tool (nickwallen) closes apache/metron#902
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/24822ddd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/24822ddd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/24822ddd
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 24822dddc68c264f59723f5e17d423cd497f6807
Parents: 35d81cb
Author: nickwallen <ni...@nickallen.org>
Authored: Thu Jan 25 18:48:02 2018 -0500
Committer: nickallen <ni...@apache.org>
Committed: Thu Jan 25 18:48:02 2018 -0500
----------------------------------------------------------------------
.github/PULL_REQUEST_TEMPLATE.md | 7 +-
.travis.yml | 2 +-
build_utils/README.md | 57 ----
build_utils/create_bundled_licenses.sh | 23 --
build_utils/generate_license.py | 108 --------
build_utils/list_dependencies.sh | 19 --
build_utils/release-utils/metron-rc-check | 269 -------------------
.../release-utils/validate-jira-for-release | 197 --------------
build_utils/verify_license.py | 44 ---
build_utils/verify_licenses.sh | 19 --
dev-utilities/build-utils/README.md | 64 +++++
.../build-utils/create_bundled_licenses.sh | 24 ++
dev-utilities/build-utils/generate_license.py | 108 ++++++++
dev-utilities/build-utils/list_dependencies.sh | 19 ++
dev-utilities/build-utils/verify_license.py | 44 +++
dev-utilities/build-utils/verify_licenses.sh | 19 ++
dev-utilities/committer-utils/README.md | 114 ++++++++
dev-utilities/committer-utils/prepare-commit | 216 +++++++++++++++
dev-utilities/release-utils/metron-rc-check | 269 +++++++++++++++++++
.../release-utils/validate-jira-for-release | 197 ++++++++++++++
site-book/bin/generate-md.sh | 5 +-
21 files changed, 1080 insertions(+), 744 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/.github/PULL_REQUEST_TEMPLATE.md
----------------------------------------------------------------------
diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md
index ee0ed83..af6c1e7 100644
--- a/.github/PULL_REQUEST_TEMPLATE.md
+++ b/.github/PULL_REQUEST_TEMPLATE.md
@@ -12,7 +12,7 @@ Please refer also to our [Build Verification Guidelines](https://cwiki.apache.or
In order to streamline the review of the contribution we ask you follow these guidelines and ask you to double check the following:
### For all changes:
-- [ ] Is there a JIRA ticket associated with this PR? If not one needs to be created at [Metron Jira](https://issues.apache.org/jira/browse/METRON/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel).
+- [ ] Is there a JIRA ticket associated with this PR? If not one needs to be created at [Metron Jira](https://issues.apache.org/jira/browse/METRON/?selectedTab=com.atlassian.jira.jira-projects-plugin:summary-panel).
- [ ] Does your PR title start with METRON-XXXX where XXXX is the JIRA number you are trying to resolve? Pay particular attention to the hyphen "-" character.
- [ ] Has your PR been rebased against the latest commit within the target branch (typically master)?
@@ -22,11 +22,11 @@ In order to streamline the review of the contribution we ask you follow these gu
- [ ] Have you included steps or a guide to how the change may be verified and tested manually?
- [ ] Have you ensured that the full suite of tests and checks have been executed in the root metron folder via:
```
- mvn -q clean integration-test install && build_utils/verify_licenses.sh
+ mvn -q clean integration-test install && dev-utilities/build-utils/verify_licenses.sh
```
- [ ] Have you written or updated unit tests and or integration tests to verify your changes?
-- [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)?
+- [ ] If adding new dependencies to the code, are these dependencies licensed in a way that is compatible for inclusion under [ASF 2.0](http://www.apache.org/legal/resolved.html#category-a)?
- [ ] Have you verified the basic functionality of the build by building and running locally with Vagrant full-dev environment or the equivalent?
### For documentation related changes:
@@ -40,4 +40,3 @@ In order to streamline the review of the contribution we ask you follow these gu
#### Note:
Please ensure that once the PR is submitted, you check travis-ci for build issues and submit an update to your PR as soon as possible.
It is also recommended that [travis-ci](https://travis-ci.org) is set up for your personal repository such that your branches are built there before submitting a pull request.
-
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/.travis.yml
----------------------------------------------------------------------
diff --git a/.travis.yml b/.travis.yml
index f5edfb2..71d7165 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -36,7 +36,7 @@ install:
- time mvn -q -T 2C -DskipTests clean install
script:
- - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh
+ - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time dev-utilities/build-utils/verify_licenses.sh
before_cache:
- rm -rf $HOME/.m2/repository/org/apache/metron
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/README.md
----------------------------------------------------------------------
diff --git a/build_utils/README.md b/build_utils/README.md
deleted file mode 100644
index 04087ee..0000000
--- a/build_utils/README.md
+++ /dev/null
@@ -1,57 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Build Utilities
-
-The aim of the build utilities project is to provide some scripting
-around the care and maintenance of the building infrastructure. At the
-moment the primary mission is around utilities that assist us manage the
-licenses of our dependencies and generate the appropriate notices or
-licenses.
-
-## `dependencies_with_url.csv`
-This file is the reference file for all of our dependencies.
-If you add a dependency, you must add a line to the
-`dependencies_with_url.csv` file.
-
-## `list_dependencies.sh`
-
-List all of the transitive dependencies for the project rooted at cwd.
-
-## `verify_licenses.sh`
-
-This script, as run by our travis build infrastructure, will look at the
-dependencies and verify that we know about them. Travis will use this
-script which takes the transitive dependency list and check against the
-`dependencies_with_url.csv` file to ensure that it's listed. This will
-make sure we track dependencies and do not have any unacceptable
-dependencies.
-
-If you want to dump all of the dependencies that it doesn't know about,
-from the top level directory:
-`build_utils/list_dependencies.sh | python build_utils/verify_license.py ./dependencies_with_url.csv dump`
-
-## `create_bundled_licenses.sh`
-
-This script is intended to regenerate the licenses for each project that
-bundles its dependencies. Because we bundle our dependencies in a
-shaded jar, we
-[must](http://www.apache.org/dev/licensing-howto.html#deps-of-deps) specify a `LICENSE` file with the permissively
-licensed dependencies notated as per [here](http://www.apache.org/dev/licensing-howto.html#permissive-deps)
-
-Example command to regenerate licenses (run from top level directory):
-`for i in $(find . -name LICENSE | grep src | grep META-INF | awk -Fsrc '{print $1}');do build_utils/create_bundled_licenses.sh $i;done`
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/create_bundled_licenses.sh
----------------------------------------------------------------------
diff --git a/build_utils/create_bundled_licenses.sh b/build_utils/create_bundled_licenses.sh
deleted file mode 100755
index 67a8126..0000000
--- a/build_utils/create_bundled_licenses.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-TLD=$(pwd)
-LICENSE_FILE="src/main/resources/META-INF/LICENSE"
-pushd $1
-$TLD/build_utils/list_dependencies.sh | python $TLD/build_utils/generate_license.py $TLD/dependencies_with_url.csv $TLD/LICENSE 1> $LICENSE_FILE
-popd
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/generate_license.py
----------------------------------------------------------------------
diff --git a/build_utils/generate_license.py b/build_utils/generate_license.py
deleted file mode 100644
index 366e515..0000000
--- a/build_utils/generate_license.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import sets
-import sys
-
-category_a_licenses = [ "BSD Software License",
- "Creative Commons License",
- "Common Development and Distribution License",
- "Common Development and Distribution License v1.1",
- "Common Development and Distribution License v1.0",
- "Common Public License v1.0",
- "Eclipse Public License v1.0",
- "MIT Software License",
- "Mozilla Public License v2.0"
- ]
-
-
-license_mapping = {
- "ASLv2": "Apache Software License v2",
- "Apache 2": "Apache Software License v2",
- "Apache 2.0": "Apache Software License v2",
- "Apache License": "Apache Software License v2",
- "Apache License 2.0": "Apache Software License v2",
- "Apache License V2.0": "Apache Software License v2",
- "Apache License Version 2.0": "Apache Software License v2",
- "Apache Software Licenses": "Apache Software License v2",
- "Apache v2": "Apache Software License v2",
- "The Apache Software License": "Apache Software License v2",
- "BSD": "BSD Software License",
- "BSD 2-clause": "BSD Software License",
- "BSD 3-Clause \"New\" or \"Revised\" License (BSD-3-Clause)" : "BSD Software License",
- "BSD 3-Clause License": "BSD Software License",
- "BSD 3-clause": "BSD Software License",
- "BSD-like": "BSD Software License",
- "CC0 1.0 Universal": "Creative Commons License",
- "CDDL": "Common Development and Distribution License",
- "CDDL 1.1": "Common Development and Distribution License v1.1",
- "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0": "Common Development and Distribution License v1.0",
- "Common Development and Distribution License (CDDL) v1.0": "Common Development and Distribution License v1.0",
- "Common Public License Version 1.0": "Common Public License v1.0",
- "Eclipse Public License 1.0": "Eclipse Public License v1.0",
- "MIT" : "MIT Software License",
- "MIT License" : "MIT Software License",
- "Mozilla Public License Version 2.0" : "Mozilla Public License v2.0",
- "New BSD License" : "BSD Software License",
- "New BSD license" : "BSD Software License",
- "Public" : "Public Domain",
- "Public Domain" : "Public Domain",
- "The BSD 3-Clause License" : "BSD Software License",
- "The BSD License" : "BSD Software License",
- "The MIT License" : "MIT Software License",
- "ACCEPTABLE" : "Ignore"
-}
-
-def read_component(i):
- with open(i, 'r') as fp:
- component_lines = fp.readlines()
- ret = {}
- for line in component_lines:
- if len(line) > 0:
- tokens = line.split(',')
- key = tokens[0]
- url = tokens[-1].strip()
- license = license_mapping[tokens[1].strip()]
- if license is None:
- raise ValueError("unable to normalize license: " + tokens[1])
- l = line.split(',')[0].strip()
- ret[key] = { 'url' : url, 'license' : license }
- return ret
-
-def read_license(f) :
- with open(f) as fp:
- return fp.read()
-
-def get_blurb(component, license_info):
- tokens = component.split(':')
- artifact_id = tokens[1]
- version = tokens[3]
- return "This product bundles " + artifact_id + " " + version + ", which is available under a \"" + license_info['license'] + "\" license. " + "For details, see " + license_info['url']
-
-if __name__ == '__main__':
- components = read_component(sys.argv[1])
- license = read_license(sys.argv[2])
- for line in sys.stdin:
- component = line.strip()
- if len(component) == 0 or component == 'none' or component not in components:
- continue
- else:
- license_info = components[component]
- if license_info['license'] in category_a_licenses:
- license = license + "\n" + get_blurb(component, license_info)
- continue
- print license
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/list_dependencies.sh
----------------------------------------------------------------------
diff --git a/build_utils/list_dependencies.sh b/build_utils/list_dependencies.sh
deleted file mode 100755
index a56a5bd..0000000
--- a/build_utils/list_dependencies.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-{ mvn dependency:list ; mvn dependency:list -PHDP-2.5.0.0 ; } | grep "^\[INFO\] " | awk '{print $2}' | grep -v "org.apache" | grep -v "test" | grep -v "provided" | grep -v "runtime" | grep -v ":system" | sort | uniq
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/release-utils/metron-rc-check
----------------------------------------------------------------------
diff --git a/build_utils/release-utils/metron-rc-check b/build_utils/release-utils/metron-rc-check
deleted file mode 100755
index 7084106..0000000
--- a/build_utils/release-utils/metron-rc-check
+++ /dev/null
@@ -1,269 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-shopt -s nocasematch
-
-function help {
- echo " "
- echo "usage: ${0}"
- echo " -v/--version=<version> The version of the metron release. [Required]"
- echo " -c/--candidate=<RC#> Defines the Release Candidate. [Required]"
- echo " -b/--bro=<bro version> The version of the bro kafka plugin. [Required]"
- echo " -h/--help Usage information."
- echo " "
- echo "example: "
- echo " metron-rc-check --version=0.4.2 --candidate=RC2 --bro=0.1.0"
- echo " "
-}
-
-METRON_DIST="https://dist.apache.org/repos/dist/dev/metron/"
-# print help, if the user just runs this without any args
-if [ "$#" -eq 0 ]; then
- help
- exit 1
-fi
-
-# handle command line options
-for i in "$@"; do
- case $i in
- #
- # VERSION: The release version of Metron to validate.
- #
- #
- -v=*|--version=*)
- VERSION="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # RC: Defines the RC# to use
- #
- # -c=RC2
- # --candidate=RC2
- #
- -c=*|--candidate=*)
- CANDIDATE="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # END: Defines the last commit to inspect
- #
- # -b=0.1.0
- # --bro=0.1.0
- #
- -b=*|--bro=*)
- BRO="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # -h/--help
- #
- -h|--help)
- help
- exit 0
- shift # past argument with no value
- ;;
-
- #
- # Unknown option
- #
- *)
- UNKNOWN_OPTION="${i#*=}"
- echo "Error: unknown option: $UNKNOWN_OPTION"
- help
- ;;
- esac
-done
-
-# validation
-if [ -z "$VERSION" ]; then
- echo "Missing -v/--version is is required"
- exit 1
-fi
-if [[ "$VERSION" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
- METRON_VERSION="$VERSION"
-else
- echo "[ERROR] "$VERSION" may not be a valid version number"
- exit 1
-fi
-
-if [ -z "$CANDIDATE" ]; then
- echo "Missing -c/--candidate which is required"
- exit 1
-fi
-
-if [[ "$CANDIDATE" =~ ^RC[0-9]+ ]]; then
- RC=$(echo "$CANDIDATE" | tr '[:upper:]' '[:lower:]')
- UPPER_RC=$(echo "$CANDIDATE" | tr '[:lower:]' '[:upper:]')
-elif [[ "$CANDIDATE" =~ ^[0-9]+ ]]; then
- RC=rc"$CANDIDATE"
- UPPER_RC=RC"$CANDIDATE"
-else
- echo "[ERROR] invalid RC, valid is RC# or just #"
- exit 1
-fi
-
-if [ -z "$BRO" ]; then
- echo "Missing -b/--bro which is required"
- exit 1
-fi
-
-if [[ "$BRO" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
- BRO_VERSION="$BRO"
-else
- echo "[ERROR] $BRO may not be a valid version number"
- exit 1
-fi
-
-echo "Metron Version $METRON_VERSION"
-echo "Release Candidate $RC"
-echo "Bro Plugin Version $BRO_VERSION"
-
-METRON_RC_DIST="$METRON_DIST$METRON_VERSION-$UPPER_RC"
-echo "Metron RC Distribution Root is $METRON_RC_DIST"
-
-# working directory
-WORK="$HOME/tmp/metron-$METRON_VERSION-$RC"
-
-# handle tilde expansion
-WORK="${WORK/#\~/$HOME}"
-
-# warn the user if the working directory exists
-if [ -d "$WORK" ]; then
- echo "[ERROR] Directory $WORK exists, please rename it and start over"
- exit 1
-fi
-
-if [ ! -d "$WORK" ]; then
- mkdir -p "$WORK"
-fi
-echo "Working directory $WORK"
-
-KEYS="$METRON_RC_DIST/KEYS"
-METRON_ASSEMBLY="$METRON_RC_DIST/apache-metron-$METRON_VERSION-$RC.tar.gz"
-METRON_ASSEMBLY_SIG="$METRON_ASSEMBLY.asc"
-METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz"
-METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc"
-
-echo "Downloading $KEYS"
-if ! wget -P "$WORK" "$KEYS" ; then
- echo "[ERROR] Failed to download $KEYS"
- exit 1
-fi
-
-echo "Downloading $METRON_ASSEMBLY"
-if ! wget -P "$WORK" "$METRON_ASSEMBLY" ; then
- echo "[ERROR] Failed to download $METRON_ASSEMBLY"
- exit 1
-fi
-echo "Downloading $METRON_ASSEMBLY_SIG"
-if ! wget -P "$WORK" "$METRON_ASSEMBLY_SIG" ; then
- echo "[ERROR] Failed to download $METRON_ASSEMBLY_SIG"
- exit 1
-fi
-echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY"
-if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY" ; then
- echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY"
- exit 1
-fi
-echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY_ASC"
-if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY_ASC" ; then
- echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY_ASC"
- exit 1
-fi
-
-cd "$WORK" || exit 1
-echo "importing metron keys"
-
-if ! gpg --import KEYS ; then
- echo "[ERROR] failed to import KEYS"
- exit 1
-fi
-
-echo "Verifying Metron Assembly"
-if ! gpg --verify ./"apache-metron-$METRON_VERSION-$RC.tar.gz.asc" "apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
- echo "[ERROR] failed to verify Metron Assembly"
- exit 1
-fi
-
-echo "Verifying Bro Kafka Plugin Assembly"
-if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
- echo "[ERROR] failed to verify Bro Kafka Plugin Assembly"
- exit 1
-fi
-
-echo "Unpacking Assemblies"
-if ! tar -xzf "apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
- echo "[ERROR] failed to unpack Metron Assembly"
- exit 1
-fi
-
-if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
- echo "[ERROR] failed to unpack Bro Kafka Plugin Assembly"
- exit 1
-fi
-
-echo ""
-echo ""
-read -p " run test suite [install, unit tests, integration tests, ui tests, licenses, rpm build]? [yN] " -n 1 -r
-echo
-DID_BUILD=0
-if [[ $REPLY =~ ^[Yy]$ ]]; then
- cd "apache-metron-$METRON_VERSION-$RC" || exit 1
- if ! mvn -q -T 2C -DskipTests clean install ; then
- echo "[ERROR] failed to mvn install metron"
- exit 1
- fi
- if ! mvn -q -T 2C surefire:test@unit-tests ; then
- echo "[ERROR] failed unit tests"
- exit 1
- fi
- if ! mvn -q surefire:test@integration-tests ; then
- echo "[ERROR] failed integration tests"
- exit 1
- fi
- if ! mvn -q test --projects metron-interface/metron-config ; then
- echo "[ERROR] failed metron-config tests"
- exit 1
- fi
- build_utils/verify_licenses.sh | tee ../build-lic.log
- cd metron-deployment || exit 1
- if ! mvn -q package -DskipTests -P build-rpms ; then
- echo "[ERROR] failed to build rpm"
- exit 1
- fi
- cd .. || exit 1
- DID_BUILD=1
-fi
-
-#ask if build test vagrant METRON
-
-# run tests?
-echo ""
-echo ""
-read -p " run vagrant full_dev? [yN] " -n 1 -r
-echo
-if [[ $REPLY =~ ^[Yy]$ ]]; then
- cd "$WORK/apache-metron-$METRON_VERSION-$RC/metron-deployment/vagrant/full-dev-platform" || exit 1
- if [[ ${DID_BUILD} -ne 1 ]]; then
- vagrant up
- else
- vagrant --ansible-skip-tags="build,sensors,quick-dev" up
- fi
-fi
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/release-utils/validate-jira-for-release
----------------------------------------------------------------------
diff --git a/build_utils/release-utils/validate-jira-for-release b/build_utils/release-utils/validate-jira-for-release
deleted file mode 100755
index df5776c..0000000
--- a/build_utils/release-utils/validate-jira-for-release
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Finds all commits since the last release tag, then ensures that each
-# is marked 'Done' and that the fix version is set to the next release.
-#
-# For example, to validate JIRA for the 0.4.2 release, you would run the
-# following command.
-#
-# validate-jira-for-release --version=0.4.2 --start=tags/apache-metron-0.4.1-release
-#
-# This will output a table containing each JIRA that was inspected along with
-# the fix version, status, and assignee. If the fix version or status is incorrect
-# a link will be printed so that the JIRA can be manually fixed. The JIRA
-# only needs to be fixed if a URL is shown.
-#
-# JIRA STATUS FIX VERSION ASSIGNEE FIX
-# METRON-1345 Done 0.4.2 Michael Miklavcic
-# METRON-1349 Done Next + 1 Nick Allen https://issues.apache.org/jira/browse/METRON-1349
-# METRON-1343 Done Mohan https://issues.apache.org/jira/browse/METRON-1343
-# ...
-#
-
-function help {
- echo " "
- echo "usage: ${0}"
- echo " -v/--version=<version> The version of the next release. [Required]"
- echo " -s/--start=<start> Defines the first commit to inspect. [Required]"
- echo " -e/--end=<end> Defines the last commit to inspect. "
- echo " -r/--repo=<repo> The Git repo to work from."
- echo " -b/--branch=<branch> The branch to work from."
- echo " -h/--help Usage information."
- echo " "
- echo "example: "
- echo " validate-jira-for-release --version=0.4.2 --start=tags/apache-metron-0.4.1-release"
- echo " "
-}
-
-# define default values
-END="HEAD"
-REPO="https://git-wip-us.apache.org/repos/asf/metron.git"
-BRANCH="master"
-
-# print help, if the user just runs this without any args
-if [ "$#" -eq 0 ]; then
- help
- exit 1
-fi
-
-# handle command line options
-for i in "$@"; do
- case $i in
- #
- # VERSION: The release version to validate; the 'next' release.
- #
- #
- -v=*|--version=*)
- VERSION="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # START: Defines the first commit to inspect
- #
- # -s=tags/apache-metron-0.4.1-release
- # --start=tags/apache-metron-0.4.1-release
- #
- -s=*|--start=*)
- START="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # END: Defines the last commit to inspect
- #
- # -e=HEAD
- # --end=HEAD
- #
- -e=*|--end=*)
- END="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # REPO: Define the Git repo to work from
- #
- # -r=https://git-wip-us.apache.org/repos/asf/metron.git
- # --repo=<repo-url>
- #
- -r=*|--repo=*)
- REPO="${i#*=}"
- shift # past argument=value
- ;;
-
- #
- # BRANCH: The branch to work from.
- #
- # -b=master
- # --branch=master
- #
- -b=*|--branch=*)
- BRANCH="${i#*=}"
- shift # past argument with no value
- ;;
-
- #
- # -h/--help
- #
- -h|--help)
- help
- exit 0
- shift # past argument with no value
- ;;
-
- #
- # Unknown option
- #
- *)
- UNKNOWN_OPTION="${i#*=}"
- echo "Error: unknown option: $UNKNOWN_OPTION"
- help
- ;;
- esac
-done
-
-WORKDIR="~/tmp"
-
-# ensure all required values are set
-if [ -z "$VERSION" ]; then
- echo "Missing -v/--version is is required"
- exit 1
-fi
-if [ -z "$START" ]; then
- echo "Missing -s/--start which is required"
- exit 1
-fi
-if [ -z "$END" ]; then
- echo "Missing -e/--end which is required"
- exit 1
-fi
-if [ -z "$REPO" ]; then
- echo "Missing -r/--repo which is required"
- exit 1
-fi
-if [ -z "$BRANCH" ]; then
- echo "Missing -b/--branch which is required"
- exit 1
-fi
-
-# clone the metron repo and fetch all tags
-git clone $REPO "metron-$VERSION"
-git checkout $BRANCH
-cd "$WORKDIR/metron-$VERSION"
-git fetch --all --tags
-
-# find all JIRAs that have been committed since the last release
-GET_JIRAS="git log $START..$END --oneline | grep -E -o 'METRON[- ]*[0-9]+'"
-
-# print the header
-FORMAT_STR="%15s %15s %15s %30s %50s\n"
-printf "$FORMAT_STR" "JIRA" "STATUS" "FIX VERSION" "ASSIGNEE" "FIX"
-
-# for each JIRA since the last release tag...
-eval $GET_JIRAS | while read JIRA ; do
-
- # fetch the JIRA content
- URL="https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml"
- CONTENT=`curl -s $URL`
-
- # painfully extract some fields
- STATUS=`echo "$CONTENT" | grep "<status[^>]*>" | sed 's/^.*<status[^>]*>//' | sed 's/<.status>.*$//'`
- ASSIGNEE=`echo "$CONTENT" | grep "<assignee[^>]*>" | sed 's/^.*<assignee[^>]*>//' | sed 's/<.assignee>.*$//'`
- FIXV=`echo "$CONTENT" | grep "<fixVersion[^>]*>" | sed 's/^.*<fixVersion[^>]*>//' | sed 's/<.fixVersion>.*$//'`
-
- # the link is only populated, if there is something to fix
- LINK=""
- if [ "$FIXV" != "$VERSION" ] || [ "$STATUS" != "Done" ]; then
- LINK="https://issues.apache.org/jira/browse/$JIRA"
- fi
-
- # show the JIRA
- printf "$FORMAT_STR" "$JIRA" "$STATUS" "$FIXV" "$ASSIGNEE" "$LINK"
-done
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/verify_license.py
----------------------------------------------------------------------
diff --git a/build_utils/verify_license.py b/build_utils/verify_license.py
deleted file mode 100644
index e9e9cfd..0000000
--- a/build_utils/verify_license.py
+++ /dev/null
@@ -1,44 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-import sets
-import sys
-
-def read_component(i):
- with open(i, 'r') as fp:
- component_lines = fp.readlines()
- ret = []
- for line in component_lines:
- if len(line) > 0:
- l = line.split(',')[0].strip()
- ret.append(l)
- return sets.Set(ret)
-
-if __name__ == '__main__':
- components = read_component(sys.argv[1])
- components_not_found = []
- for line in sys.stdin:
- component = line.strip()
- if len(component) == 0 or component == 'none' or component in components:
- continue
- else:
- if len(sys.argv) > 2:
- print component
- else:
- components_not_found.append(component)
- if len(components_not_found) > 0:
- raise ValueError("Unable to find these components: \n " + "\n ".join(components_not_found) + "\nin the acceptable list of components: " + sys.argv[1])
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/build_utils/verify_licenses.sh
----------------------------------------------------------------------
diff --git a/build_utils/verify_licenses.sh b/build_utils/verify_licenses.sh
deleted file mode 100755
index 5b1d1ee..0000000
--- a/build_utils/verify_licenses.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-build_utils/list_dependencies.sh | python build_utils/verify_license.py ./dependencies_with_url.csv
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/build-utils/README.md
----------------------------------------------------------------------
diff --git a/dev-utilities/build-utils/README.md b/dev-utilities/build-utils/README.md
new file mode 100644
index 0000000..495a256
--- /dev/null
+++ b/dev-utilities/build-utils/README.md
@@ -0,0 +1,64 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Build Utilities
+
+The aim of the build utilities project is to provide some scripting
+around the care and maintenance of the building infrastructure. At the
+moment the primary mission is around utilities that assist us manage the
+licenses of our dependencies and generate the appropriate notices or
+licenses.
+
+## `dependencies_with_url.csv`
+This file is the reference file for all of our dependencies.
+If you add a dependency, you must add a line to the
+`dependencies_with_url.csv` file.
+
+## `list_dependencies.sh`
+
+List all of the transitive dependencies for the project rooted at cwd.
+
+## `verify_licenses.sh`
+
+This script, as run by our travis build infrastructure, will look at the
+dependencies and verify that we know about them. Travis will use this
+script which takes the transitive dependency list and check against the
+`dependencies_with_url.csv` file to ensure that it's listed. This will
+make sure we track dependencies and do not have any unacceptable
+dependencies.
+
+If you want to dump all of the dependencies that it doesn't know about,
+from the top level directory:
+
+```
+dev-utilities/build-utils/list_dependencies.sh | python dev-utilities/build-utils/verify_license.py ./dependencies_with_url.csv dump
+```
+
+
+## `create_bundled_licenses.sh`
+
+This script is intended to regenerate the licenses for each project that
+bundles its dependencies. Because we bundle our dependencies in a
+shaded jar, we
+[must](http://www.apache.org/dev/licensing-howto.html#deps-of-deps) specify a `LICENSE` file with the permissively
+licensed dependencies notated as per [here](http://www.apache.org/dev/licensing-howto.html#permissive-deps)
+
+Example command to regenerate licenses (run from top level directory):
+
+```
+for i in $(find . -name LICENSE | grep src | grep META-INF | awk -Fsrc '{print $1}');do dev-utilities/build-utils/create_bundled_licenses.sh $i;done
+```
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/build-utils/create_bundled_licenses.sh
----------------------------------------------------------------------
diff --git a/dev-utilities/build-utils/create_bundled_licenses.sh b/dev-utilities/build-utils/create_bundled_licenses.sh
new file mode 100755
index 0000000..b853bad
--- /dev/null
+++ b/dev-utilities/build-utils/create_bundled_licenses.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+TLD=$(pwd)
+LICENSE_FILE="src/main/resources/META-INF/LICENSE"
+pushd $1
+$TLD/dev-utilities/build-utils/list_dependencies.sh |
+ python $TLD/dev-utilities/build-utils/generate_license.py $TLD/dependencies_with_url.csv $TLD/LICENSE 1> $LICENSE_FILE
+popd
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/build-utils/generate_license.py
----------------------------------------------------------------------
diff --git a/dev-utilities/build-utils/generate_license.py b/dev-utilities/build-utils/generate_license.py
new file mode 100644
index 0000000..366e515
--- /dev/null
+++ b/dev-utilities/build-utils/generate_license.py
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sets
+import sys
+
+category_a_licenses = [ "BSD Software License",
+ "Creative Commons License",
+ "Common Development and Distribution License",
+ "Common Development and Distribution License v1.1",
+ "Common Development and Distribution License v1.0",
+ "Common Public License v1.0",
+ "Eclipse Public License v1.0",
+ "MIT Software License",
+ "Mozilla Public License v2.0"
+ ]
+
+
+license_mapping = {
+ "ASLv2": "Apache Software License v2",
+ "Apache 2": "Apache Software License v2",
+ "Apache 2.0": "Apache Software License v2",
+ "Apache License": "Apache Software License v2",
+ "Apache License 2.0": "Apache Software License v2",
+ "Apache License V2.0": "Apache Software License v2",
+ "Apache License Version 2.0": "Apache Software License v2",
+ "Apache Software Licenses": "Apache Software License v2",
+ "Apache v2": "Apache Software License v2",
+ "The Apache Software License": "Apache Software License v2",
+ "BSD": "BSD Software License",
+ "BSD 2-clause": "BSD Software License",
+ "BSD 3-Clause \"New\" or \"Revised\" License (BSD-3-Clause)" : "BSD Software License",
+ "BSD 3-Clause License": "BSD Software License",
+ "BSD 3-clause": "BSD Software License",
+ "BSD-like": "BSD Software License",
+ "CC0 1.0 Universal": "Creative Commons License",
+ "CDDL": "Common Development and Distribution License",
+ "CDDL 1.1": "Common Development and Distribution License v1.1",
+ "COMMON DEVELOPMENT AND DISTRIBUTION LICENSE (CDDL) Version 1.0": "Common Development and Distribution License v1.0",
+ "Common Development and Distribution License (CDDL) v1.0": "Common Development and Distribution License v1.0",
+ "Common Public License Version 1.0": "Common Public License v1.0",
+ "Eclipse Public License 1.0": "Eclipse Public License v1.0",
+ "MIT" : "MIT Software License",
+ "MIT License" : "MIT Software License",
+ "Mozilla Public License Version 2.0" : "Mozilla Public License v2.0",
+ "New BSD License" : "BSD Software License",
+ "New BSD license" : "BSD Software License",
+ "Public" : "Public Domain",
+ "Public Domain" : "Public Domain",
+ "The BSD 3-Clause License" : "BSD Software License",
+ "The BSD License" : "BSD Software License",
+ "The MIT License" : "MIT Software License",
+ "ACCEPTABLE" : "Ignore"
+}
+
+def read_component(i):
+ with open(i, 'r') as fp:
+ component_lines = fp.readlines()
+ ret = {}
+ for line in component_lines:
+ if len(line) > 0:
+ tokens = line.split(',')
+ key = tokens[0]
+ url = tokens[-1].strip()
+ license = license_mapping[tokens[1].strip()]
+ if license is None:
+ raise ValueError("unable to normalize license: " + tokens[1])
+ l = line.split(',')[0].strip()
+ ret[key] = { 'url' : url, 'license' : license }
+ return ret
+
+def read_license(f) :
+ with open(f) as fp:
+ return fp.read()
+
+def get_blurb(component, license_info):
+ tokens = component.split(':')
+ artifact_id = tokens[1]
+ version = tokens[3]
+ return "This product bundles " + artifact_id + " " + version + ", which is available under a \"" + license_info['license'] + "\" license. " + "For details, see " + license_info['url']
+
+if __name__ == '__main__':
+ components = read_component(sys.argv[1])
+ license = read_license(sys.argv[2])
+ for line in sys.stdin:
+ component = line.strip()
+ if len(component) == 0 or component == 'none' or component not in components:
+ continue
+ else:
+ license_info = components[component]
+ if license_info['license'] in category_a_licenses:
+ license = license + "\n" + get_blurb(component, license_info)
+ continue
+ print license
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/build-utils/list_dependencies.sh
----------------------------------------------------------------------
diff --git a/dev-utilities/build-utils/list_dependencies.sh b/dev-utilities/build-utils/list_dependencies.sh
new file mode 100755
index 0000000..a56a5bd
--- /dev/null
+++ b/dev-utilities/build-utils/list_dependencies.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{ mvn dependency:list ; mvn dependency:list -PHDP-2.5.0.0 ; } | grep "^\[INFO\] " | awk '{print $2}' | grep -v "org.apache" | grep -v "test" | grep -v "provided" | grep -v "runtime" | grep -v ":system" | sort | uniq
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/build-utils/verify_license.py
----------------------------------------------------------------------
diff --git a/dev-utilities/build-utils/verify_license.py b/dev-utilities/build-utils/verify_license.py
new file mode 100644
index 0000000..e9e9cfd
--- /dev/null
+++ b/dev-utilities/build-utils/verify_license.py
@@ -0,0 +1,44 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sets
+import sys
+
+def read_component(i):
+ with open(i, 'r') as fp:
+ component_lines = fp.readlines()
+ ret = []
+ for line in component_lines:
+ if len(line) > 0:
+ l = line.split(',')[0].strip()
+ ret.append(l)
+ return sets.Set(ret)
+
+if __name__ == '__main__':
+ components = read_component(sys.argv[1])
+ components_not_found = []
+ for line in sys.stdin:
+ component = line.strip()
+ if len(component) == 0 or component == 'none' or component in components:
+ continue
+ else:
+ if len(sys.argv) > 2:
+ print component
+ else:
+ components_not_found.append(component)
+ if len(components_not_found) > 0:
+ raise ValueError("Unable to find these components: \n " + "\n ".join(components_not_found) + "\nin the acceptable list of components: " + sys.argv[1])
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/build-utils/verify_licenses.sh
----------------------------------------------------------------------
diff --git a/dev-utilities/build-utils/verify_licenses.sh b/dev-utilities/build-utils/verify_licenses.sh
new file mode 100755
index 0000000..ffd6e48
--- /dev/null
+++ b/dev-utilities/build-utils/verify_licenses.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+dev-utilities/build-utils/list_dependencies.sh | python dev-utilities/build-utils/verify_license.py ./dependencies_with_url.csv
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/committer-utils/README.md
----------------------------------------------------------------------
diff --git a/dev-utilities/committer-utils/README.md b/dev-utilities/committer-utils/README.md
new file mode 100644
index 0000000..60e1d51
--- /dev/null
+++ b/dev-utilities/committer-utils/README.md
@@ -0,0 +1,114 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Committer Tools
+
+This project contains tools to assist Apache Metron project committers.
+
+## Prepare Commit
+
+This script automates the process of merging a pull request into `apache/master`. The script will prompt for the pull request number. Most of the remaining information is automatically extracted from Github or the Apache JIRA.
+
+When prompted the `[value in brackets]` is used by default. To accept the default, simply press `enter`. If you would like to change the default, type it in and hit `enter` when done.
+
+In the following example, I enter the pull request number when prompted. Using the pull request number, the script can extract most of the remaining required information.
+
+1. Execute the script.
+
+ The first time the script is run, you will be prompted for additional information including your Apache username, Apache email, and Github username. These values are persisted in `~/.metron-prepare-commit`. Subsequent executions of the script will retrieve these values, rather than prompting you again for them.
+
+ ```
+ $ prepare-commit
+ your github username []: nickwallen
+ your apache userid []: nickallen
+ your apache email [nickallen@apache.org]:
+ ```
+
+1. Enter the Github pull request number.
+
+ ```
+ pull request: 897
+ local working directory [/Users/nallen/tmp/metron-pr897]:
+ origin repo [https://github.com/apache/metron]:
+
+ Cloning into '/Users/nallen/tmp/metron-pr897'...
+ remote: Counting objects: 36277, done.
+ remote: Compressing objects: 100% (108/108), done.
+ remote: Total 36277 (delta 38), reused 54 (delta 20), pack-reused 36138
+ Receiving objects: 100% (36277/36277), 57.85 MiB | 7.36 MiB/s, done.
+ Resolving deltas: 100% (13653/13653), done.
+ From https://git-wip-us.apache.org/repos/asf/metron
+ * branch master -> FETCH_HEAD
+ * [new branch] master -> upstream/master
+ Already on 'master'
+ Your branch is up to date with 'origin/master'.
+ Already up to date.
+ remote: Counting objects: 5, done.
+ remote: Total 5 (delta 3), reused 3 (delta 3), pack-reused 2
+ Unpacking objects: 100% (5/5), done.
+ From https://github.com/apache/metron
+ * [new ref] refs/pull/897/head -> pr-897
+ ```
+
+1. Enter contribution details.
+
+ The contributor's username, email, along with information about the associated Apache JIRA is extracted from the commit history.
+
+ ```
+ github contributor's username [MohanDV]:
+ github contributor's email [mohan.dv@gmail.com]:
+ issue identifier in jira [METRON-1395]:
+ issue description [Documentation missing for Produce a message to a Kafka topic Rest API endpoint]:
+ commit message [METRON-1395 Documentation missing for Produce a message to a Kafka topic Rest API endpoint (MohanDV via nickwallen) closes apache/metron#897]:
+ ```
+
+1. The contribution is then merged with master as a single commit. The changes that have been made along with the commit message are displayed.
+
+ ```
+ Squash commit -- not updating HEAD
+ Automatic merge went well; stopped before committing as requested
+ [master 998f7915] METRON-1410 Some more upgrade fallout... Can't restart Metron Indexing. (ottobackwards via nickwallen) closes apache/metron#901
+ Author: ottobackwards <ot...@gmail.com>
+ 3 files changed, 3 insertions(+), 3 deletions(-)
+
+
+ .../metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py | 2 +-
+ .../ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py | 2 +-
+ .../ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py | 2 +-
+ 3 files changed, 3 insertions(+), 3 deletions(-)
+
+ 998f7915 (HEAD -> master) METRON-1410 Cannot restart Metron Indexing. (ottobackwards via nickwallen) closes apache/metron#901
+ ```
+
+1. Run the test suite.
+
+ After the merge is complete, the script will prompt you to run the test suite. By default this is skipped, but by typing 'y' the test suite will be run.
+
+ ```
+ run test suite? [yN]
+ ```
+
+1. Finalize the changes.
+
+ To this point changes have only been made to your local repository. The script itself will not push changes to Apache. You are given instructions on how to do so. Review the summary and enter `y` at the prompt, if you are satisfied. If you are not happy, simply start over.
+
+ ```
+ Review commit carefully then run...
+ cd /Users/nallen/tmp/metron-pr897
+ git push upstream master
+ ```
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/committer-utils/prepare-commit
----------------------------------------------------------------------
diff --git a/dev-utilities/committer-utils/prepare-commit b/dev-utilities/committer-utils/prepare-commit
new file mode 100755
index 0000000..90852c0
--- /dev/null
+++ b/dev-utilities/committer-utils/prepare-commit
@@ -0,0 +1,216 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# not likely to change
+UPSTREAM=https://git-wip-us.apache.org/repos/asf/metron.git
+BASE_BRANCH=master
+CONFIG_FILE=~/.metron-prepare-commit
+GITHUB_REMOTE="origin"
+
+# does a config file already exist?
+if [ -f $CONFIG_FILE ]; then
+ . $CONFIG_FILE
+ echo " ...using settings from $CONFIG_FILE"
+fi
+
+# github account of committer (you)
+if [ -z "$GITHUB_NAME" ]; then
+ read -p " your github username [$GITHUB_NAME]: " INPUT
+ [ -n "$INPUT" ] && GITHUB_NAME=$INPUT
+
+ # write setting to config file
+ echo "GITHUB_NAME=$GITHUB_NAME" >> $CONFIG_FILE
+fi
+
+# apache id of committer (you)
+if [ -z "$APACHE_NAME" ]; then
+ read -p " your apache userid [$APACHE_NAME]: " INPUT
+ [ -n "$INPUT" ] && APACHE_NAME=$INPUT
+
+ # write setting to config file
+ echo "APACHE_NAME=$APACHE_NAME" >> $CONFIG_FILE
+fi
+
+# apache email addr of committer (you)
+if [ -z "$APACHE_EMAIL" ]; then
+ APACHE_EMAIL=${APACHE_NAME}@apache.org
+ read -p " your apache email [$APACHE_EMAIL]: " INPUT
+ [ -n "$INPUT" ] && APACHE_EMAIL=$INPUT
+
+ # write setting to config file, so it is not needed next time
+ echo "APACHE_EMAIL=$APACHE_EMAIL" >> $CONFIG_FILE
+fi
+
+# retrieve the pull request identifier
+read -p " pull request: " PR
+if [ -z "$PR" ]; then
+ echo "Error: missing pr"
+ exit 1
+fi
+
+# ensure that the pull request exists
+PR_EXISTS=`curl -sI https://api.github.com/repos/apache/metron/pulls/$PR | grep Status: | sed 's/[^0-9]//g'`
+if [ "$PR_EXISTS" != "200" ]; then
+ echo "Error: pull request #$PR does not exist"
+ exit 1
+fi
+
+# working directory
+WORK=~/tmp/metron-pr$PR
+read -p " local working directory [$WORK]: " INPUT
+[ -n "$INPUT" ] && WORK=$INPUT
+
+# handle tilde expansion
+WORK="${WORK/#\~/$HOME}"
+
+# warn the user if the working directory exists
+if [ -d "$WORK" ]; then
+ read -p " directory exists [$WORK]. continue merge on existing repo? [yN] " -n 1 -r
+ echo
+ if [[ ! $REPLY =~ ^[Yy]$ ]]; then
+ exit 1
+ fi
+fi
+
+# if working directory does not exist, checkout the base branch
+if [ ! -d "$WORK" ]; then
+
+ # origin repository
+ ORIGIN="https://github.com/apache/metron"
+ read -p " origin repo [$ORIGIN]: " INPUT
+ [ -n "$INPUT" ] && ORIGIN=$INPUT
+
+ # clone the repository and fetch updates
+ mkdir -p $WORK
+ git clone $ORIGIN $WORK
+ cd $WORK
+
+ # setup the git user and email for your apache account
+ git config user.name "$APACHE_NAME"
+ git config user.email $APACHE_EMAIL
+
+ # fetch any changes from upstream
+ git remote add upstream $UPSTREAM
+ git fetch upstream $BASE_BRANCH
+
+ # merge any changes from upstream
+ git checkout $BASE_BRANCH
+ git merge upstream/$BASE_BRANCH
+
+else
+
+ # if the repo already exists, allow the user to provide the name of the Github remote
+ # this is needed to checkout the code for the PR
+ read -p " name of github remote [$GITHUB_REMOTE]: " INPUT
+ [ -n "$INPUT" ] && GITHUB_REMOTE=$INPUT
+
+fi
+
+PR_BRANCH_REF="pull/$PR/head:pr-$PR"
+PR_BRANCH="pr-$PR"
+cd $WORK
+git fetch $GITHUB_REMOTE $PR_BRANCH_REF
+echo ""
+
+# use github api to retrieve the contributor's login
+USER=`curl -s https://api.github.com/repos/apache/metron/pulls/$PR | grep login | head -1 | awk -F":" '{print $2}' | sed 's/[^a-zA-Z.@_-]//g'`
+read -p " github contributor's username [$USER]: " INPUT
+[ -n "$INPUT" ] && USER=$INPUT
+
+# validate the github contributor
+if [ -z "$USER" ]; then
+ echo "Error: missing username"
+ exit 1
+fi
+
+# retrieve the contributor's email from the git commit history
+EMAIL=`git log $PR_BRANCH | grep Author | head -1 | awk -F"<" '{print $2}' | sed 's/[<>]//g'`
+read -p " github contributor's email [$EMAIL]: " INPUT
+[ -n "$INPUT" ] && EMAIL=$INPUT
+
+# validate email
+if [ -z "$EMAIL" ] || [ "$EMAIL" = "null" ]; then
+ echo "Error: missing email"
+ exit 1
+fi
+
+# can we extract the JIRA from the PR title?
+JIRA=`curl -s https://api.github.com/repos/apache/metron/pulls/$PR | grep title | head -1 | egrep -o -i 'METRON-[0-9]+' | awk '{print toupper($0)}'`
+read -p " issue identifier in jira [$JIRA]: " INPUT
+[ -n "$INPUT" ] && JIRA=$INPUT
+
+# validate the JIRA issue
+if [ -z "$JIRA" ]; then
+ echo "Error: missing jira"
+ exit 1
+fi
+
+# attempt to use the jira api to get a description of the jira
+DESC=`curl -s https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml | grep "<summary>" | sed 's/^.*<summary>//' | sed 's/<.summary>.*$//'`
+read -p " issue description [$DESC]: " INPUT
+[ -n "$INPUT" ] && DESC=$INPUT
+
+# validate description
+if [ -z "$DESC" ]; then
+ echo "Error: missing description"
+ exit 1
+fi
+
+# commit message
+AUTHOR="$USER <$EMAIL>"
+if [ "$USER" == "$GITHUB_NAME" ]; then
+ MSG="$JIRA $DESC ($USER) closes apache/metron#$PR"
+else
+ MSG="$JIRA $DESC ($USER via $GITHUB_NAME) closes apache/metron#$PR"
+fi
+read -p " commit message [$MSG]: " INPUT
+[ -n "$INPUT" ] && MSG=$INPUT
+
+# merge the contributor's branch and commit
+echo ""
+if git merge --squash "$PR_BRANCH"; then
+ git commit --author="$AUTHOR" -a -m "$MSG"
+else
+ exit $?
+fi
+
+# review the commit
+echo ""
+echo ""
+git diff --stat --color "upstream/$BASE_BRANCH..$BASE_BRANCH"
+echo ""
+echo ""
+git log --oneline "$BASE_BRANCH" "^upstream/$BASE_BRANCH"
+
+# run tests?
+echo ""
+echo ""
+read -p " run test suite? [yN] " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+ mvn -q -T 2C -DskipTests clean install &&
+ mvn -q -T 2C surefire:test@unit-tests &&
+ mvn -q surefire:test@integration-tests &&
+ mvn -q test --projects metron-interface/metron-config &&
+ dev-utilities/build-utils/verify_licenses.sh
+fi
+
+echo ""
+echo "Review commit carefully then run..."
+echo " cd $WORK"
+echo " git push upstream master"
+echo ""
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/release-utils/metron-rc-check
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/metron-rc-check b/dev-utilities/release-utils/metron-rc-check
new file mode 100755
index 0000000..da5412e
--- /dev/null
+++ b/dev-utilities/release-utils/metron-rc-check
@@ -0,0 +1,269 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+shopt -s nocasematch
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " -v/--version=<version> The version of the metron release. [Required]"
+ echo " -c/--candidate=<RC#> Defines the Release Candidate. [Required]"
+ echo " -b/--bro=<bro version> The version of the bro kafka plugin. [Required]"
+ echo " -h/--help Usage information."
+ echo " "
+ echo "example: "
+ echo " metron-rc-check --version=0.4.2 --candidate=RC2 --bro=0.1.0"
+ echo " "
+}
+
+METRON_DIST="https://dist.apache.org/repos/dist/dev/metron/"
+# print help, if the user just runs this without any args
+if [ "$#" -eq 0 ]; then
+ help
+ exit 1
+fi
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # VERSION: The release version of Metron to validate.
+ #
+ #
+ -v=*|--version=*)
+ VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # RC: Defines the RC# to use
+ #
+ # -c=RC2
+ # --candidate=RC2
+ #
+ -c=*|--candidate=*)
+ CANDIDATE="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # END: Defines the last commit to inspect
+ #
+ # -b=0.1.0
+ # --bro=0.1.0
+ #
+ -b=*|--bro=*)
+ BRO="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h|--help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+# validation
+if [ -z "$VERSION" ]; then
+ echo "Missing -v/--version is is required"
+ exit 1
+fi
+if [[ "$VERSION" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
+ METRON_VERSION="$VERSION"
+else
+ echo "[ERROR] "$VERSION" may not be a valid version number"
+ exit 1
+fi
+
+if [ -z "$CANDIDATE" ]; then
+ echo "Missing -c/--candidate which is required"
+ exit 1
+fi
+
+if [[ "$CANDIDATE" =~ ^RC[0-9]+ ]]; then
+ RC=$(echo "$CANDIDATE" | tr '[:upper:]' '[:lower:]')
+ UPPER_RC=$(echo "$CANDIDATE" | tr '[:lower:]' '[:upper:]')
+elif [[ "$CANDIDATE" =~ ^[0-9]+ ]]; then
+ RC=rc"$CANDIDATE"
+ UPPER_RC=RC"$CANDIDATE"
+else
+ echo "[ERROR] invalid RC, valid is RC# or just #"
+ exit 1
+fi
+
+if [ -z "$BRO" ]; then
+ echo "Missing -b/--bro which is required"
+ exit 1
+fi
+
+if [[ "$BRO" =~ ^[0-9]{1,2}\.[0-9]{1,2}\.[0-9]{1,2} ]]; then
+ BRO_VERSION="$BRO"
+else
+ echo "[ERROR] $BRO may not be a valid version number"
+ exit 1
+fi
+
+echo "Metron Version $METRON_VERSION"
+echo "Release Candidate $RC"
+echo "Bro Plugin Version $BRO_VERSION"
+
+METRON_RC_DIST="$METRON_DIST$METRON_VERSION-$UPPER_RC"
+echo "Metron RC Distribution Root is $METRON_RC_DIST"
+
+# working directory
+WORK="$HOME/tmp/metron-$METRON_VERSION-$RC"
+
+# handle tilde expansion
+WORK="${WORK/#\~/$HOME}"
+
+# warn the user if the working directory exists
+if [ -d "$WORK" ]; then
+ echo "[ERROR] Directory $WORK exists, please rename it and start over"
+ exit 1
+fi
+
+if [ ! -d "$WORK" ]; then
+ mkdir -p "$WORK"
+fi
+echo "Working directory $WORK"
+
+KEYS="$METRON_RC_DIST/KEYS"
+METRON_ASSEMBLY="$METRON_RC_DIST/apache-metron-$METRON_VERSION-$RC.tar.gz"
+METRON_ASSEMBLY_SIG="$METRON_ASSEMBLY.asc"
+METRON_KAFKA_BRO_ASSEMBLY="$METRON_RC_DIST/apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz"
+METRON_KAFKA_BRO_ASSEMBLY_ASC="$METRON_KAFKA_BRO_ASSEMBLY.asc"
+
+echo "Downloading $KEYS"
+if ! wget -P "$WORK" "$KEYS" ; then
+ echo "[ERROR] Failed to download $KEYS"
+ exit 1
+fi
+
+echo "Downloading $METRON_ASSEMBLY"
+if ! wget -P "$WORK" "$METRON_ASSEMBLY" ; then
+ echo "[ERROR] Failed to download $METRON_ASSEMBLY"
+ exit 1
+fi
+echo "Downloading $METRON_ASSEMBLY_SIG"
+if ! wget -P "$WORK" "$METRON_ASSEMBLY_SIG" ; then
+ echo "[ERROR] Failed to download $METRON_ASSEMBLY_SIG"
+ exit 1
+fi
+echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY"
+if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY" ; then
+ echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY"
+ exit 1
+fi
+echo "Downloading $METRON_KAFKA_BRO_ASSEMBLY_ASC"
+if ! wget -P "$WORK" "$METRON_KAFKA_BRO_ASSEMBLY_ASC" ; then
+ echo "[ERROR] Failed to download $METRON_KAFKA_BRO_ASSEMBLY_ASC"
+ exit 1
+fi
+
+cd "$WORK" || exit 1
+echo "importing metron keys"
+
+if ! gpg --import KEYS ; then
+ echo "[ERROR] failed to import KEYS"
+ exit 1
+fi
+
+echo "Verifying Metron Assembly"
+if ! gpg --verify ./"apache-metron-$METRON_VERSION-$RC.tar.gz.asc" "apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
+ echo "[ERROR] failed to verify Metron Assembly"
+ exit 1
+fi
+
+echo "Verifying Bro Kafka Plugin Assembly"
+if ! gpg --verify ./"apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz.asc" "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+ echo "[ERROR] failed to verify Bro Kafka Plugin Assembly"
+ exit 1
+fi
+
+echo "Unpacking Assemblies"
+if ! tar -xzf "apache-metron-$METRON_VERSION-$RC.tar.gz" ; then
+ echo "[ERROR] failed to unpack Metron Assembly"
+ exit 1
+fi
+
+if ! tar -xzf "apache-metron-bro-plugin-kafka_$BRO_VERSION.tar.gz" ; then
+ echo "[ERROR] failed to unpack Bro Kafka Plugin Assembly"
+ exit 1
+fi
+
+echo ""
+echo ""
+read -p " run test suite [install, unit tests, integration tests, ui tests, licenses, rpm build]? [yN] " -n 1 -r
+echo
+DID_BUILD=0
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+ cd "apache-metron-$METRON_VERSION-$RC" || exit 1
+ if ! mvn -q -T 2C -DskipTests clean install ; then
+ echo "[ERROR] failed to mvn install metron"
+ exit 1
+ fi
+ if ! mvn -q -T 2C surefire:test@unit-tests ; then
+ echo "[ERROR] failed unit tests"
+ exit 1
+ fi
+ if ! mvn -q surefire:test@integration-tests ; then
+ echo "[ERROR] failed integration tests"
+ exit 1
+ fi
+ if ! mvn -q test --projects metron-interface/metron-config ; then
+ echo "[ERROR] failed metron-config tests"
+ exit 1
+ fi
+ dev-utilities/build-utils/verify_licenses.sh | tee ../build-lic.log
+ cd metron-deployment || exit 1
+ if ! mvn -q package -DskipTests -P build-rpms ; then
+ echo "[ERROR] failed to build rpm"
+ exit 1
+ fi
+ cd .. || exit 1
+ DID_BUILD=1
+fi
+
+#ask if build test vagrant METRON
+
+# run tests?
+echo ""
+echo ""
+read -p " run vagrant full_dev? [yN] " -n 1 -r
+echo
+if [[ $REPLY =~ ^[Yy]$ ]]; then
+ cd "$WORK/apache-metron-$METRON_VERSION-$RC/metron-deployment/vagrant/full-dev-platform" || exit 1
+ if [[ ${DID_BUILD} -ne 1 ]]; then
+ vagrant up
+ else
+ vagrant --ansible-skip-tags="build,sensors,quick-dev" up
+ fi
+fi
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/dev-utilities/release-utils/validate-jira-for-release
----------------------------------------------------------------------
diff --git a/dev-utilities/release-utils/validate-jira-for-release b/dev-utilities/release-utils/validate-jira-for-release
new file mode 100755
index 0000000..df5776c
--- /dev/null
+++ b/dev-utilities/release-utils/validate-jira-for-release
@@ -0,0 +1,197 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Finds all commits since the last release tag, then ensures that each
+# is marked 'Done' and that the fix version is set to the next release.
+#
+# For example, to validate JIRA for the 0.4.2 release, you would run the
+# following command.
+#
+# validate-jira-for-release --version=0.4.2 --start=tags/apache-metron-0.4.1-release
+#
+# This will output a table containing each JIRA that was inspected along with
+# the fix version, status, and assignee. If the fix version or status is incorrect
+# a link will be printed so that the JIRA can be manually fixed. The JIRA
+# only needs to be fixed if a URL is shown.
+#
+# JIRA STATUS FIX VERSION ASSIGNEE FIX
+# METRON-1345 Done 0.4.2 Michael Miklavcic
+# METRON-1349 Done Next + 1 Nick Allen https://issues.apache.org/jira/browse/METRON-1349
+# METRON-1343 Done Mohan https://issues.apache.org/jira/browse/METRON-1343
+# ...
+#
+
+function help {
+ echo " "
+ echo "usage: ${0}"
+ echo " -v/--version=<version> The version of the next release. [Required]"
+ echo " -s/--start=<start> Defines the first commit to inspect. [Required]"
+ echo " -e/--end=<end> Defines the last commit to inspect. "
+ echo " -r/--repo=<repo> The Git repo to work from."
+ echo " -b/--branch=<branch> The branch to work from."
+ echo " -h/--help Usage information."
+ echo " "
+ echo "example: "
+ echo " validate-jira-for-release --version=0.4.2 --start=tags/apache-metron-0.4.1-release"
+ echo " "
+}
+
+# define default values
+END="HEAD"
+REPO="https://git-wip-us.apache.org/repos/asf/metron.git"
+BRANCH="master"
+
+# print help, if the user just runs this without any args
+if [ "$#" -eq 0 ]; then
+ help
+ exit 1
+fi
+
+# handle command line options
+for i in "$@"; do
+ case $i in
+ #
+ # VERSION: The release version to validate; the 'next' release.
+ #
+ #
+ -v=*|--version=*)
+ VERSION="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # START: Defines the first commit to inspect
+ #
+ # -s=tags/apache-metron-0.4.1-release
+ # --start=tags/apache-metron-0.4.1-release
+ #
+ -s=*|--start=*)
+ START="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # END: Defines the last commit to inspect
+ #
+ # -e=HEAD
+ # --end=HEAD
+ #
+ -e=*|--end=*)
+ END="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # REPO: Define the Git repo to work from
+ #
+ # -r=https://git-wip-us.apache.org/repos/asf/metron.git
+ # --repo=<repo-url>
+ #
+ -r=*|--repo=*)
+ REPO="${i#*=}"
+ shift # past argument=value
+ ;;
+
+ #
+ # BRANCH: The branch to work from.
+ #
+ # -b=master
+ # --branch=master
+ #
+ -b=*|--branch=*)
+ BRANCH="${i#*=}"
+ shift # past argument with no value
+ ;;
+
+ #
+ # -h/--help
+ #
+ -h|--help)
+ help
+ exit 0
+ shift # past argument with no value
+ ;;
+
+ #
+ # Unknown option
+ #
+ *)
+ UNKNOWN_OPTION="${i#*=}"
+ echo "Error: unknown option: $UNKNOWN_OPTION"
+ help
+ ;;
+ esac
+done
+
+WORKDIR="~/tmp"
+
+# ensure all required values are set
+if [ -z "$VERSION" ]; then
+ echo "Missing -v/--version is is required"
+ exit 1
+fi
+if [ -z "$START" ]; then
+ echo "Missing -s/--start which is required"
+ exit 1
+fi
+if [ -z "$END" ]; then
+ echo "Missing -e/--end which is required"
+ exit 1
+fi
+if [ -z "$REPO" ]; then
+ echo "Missing -r/--repo which is required"
+ exit 1
+fi
+if [ -z "$BRANCH" ]; then
+ echo "Missing -b/--branch which is required"
+ exit 1
+fi
+
+# clone the metron repo and fetch all tags
+git clone $REPO "metron-$VERSION"
+git checkout $BRANCH
+cd "$WORKDIR/metron-$VERSION"
+git fetch --all --tags
+
+# find all JIRAs that have been committed since the last release
+GET_JIRAS="git log $START..$END --oneline | grep -E -o 'METRON[- ]*[0-9]+'"
+
+# print the header
+FORMAT_STR="%15s %15s %15s %30s %50s\n"
+printf "$FORMAT_STR" "JIRA" "STATUS" "FIX VERSION" "ASSIGNEE" "FIX"
+
+# for each JIRA since the last release tag...
+eval $GET_JIRAS | while read JIRA ; do
+
+ # fetch the JIRA content
+ URL="https://issues.apache.org/jira/si/jira.issueviews:issue-xml/$JIRA/$JIRA.xml"
+ CONTENT=`curl -s $URL`
+
+ # painfully extract some fields
+ STATUS=`echo "$CONTENT" | grep "<status[^>]*>" | sed 's/^.*<status[^>]*>//' | sed 's/<.status>.*$//'`
+ ASSIGNEE=`echo "$CONTENT" | grep "<assignee[^>]*>" | sed 's/^.*<assignee[^>]*>//' | sed 's/<.assignee>.*$//'`
+ FIXV=`echo "$CONTENT" | grep "<fixVersion[^>]*>" | sed 's/^.*<fixVersion[^>]*>//' | sed 's/<.fixVersion>.*$//'`
+
+ # the link is only populated, if there is something to fix
+ LINK=""
+ if [ "$FIXV" != "$VERSION" ] || [ "$STATUS" != "Done" ]; then
+ LINK="https://issues.apache.org/jira/browse/$JIRA"
+ fi
+
+ # show the JIRA
+ printf "$FORMAT_STR" "$JIRA" "$STATUS" "$FIXV" "$ASSIGNEE" "$LINK"
+done
http://git-wip-us.apache.org/repos/asf/metron/blob/24822ddd/site-book/bin/generate-md.sh
----------------------------------------------------------------------
diff --git a/site-book/bin/generate-md.sh b/site-book/bin/generate-md.sh
index 818ab99..464cb69 100755
--- a/site-book/bin/generate-md.sh
+++ b/site-book/bin/generate-md.sh
@@ -19,7 +19,7 @@
# ------------------------------------------------------------------
#
# This script collects the *.md files and other resources needed to generate
-# a book-like collection of end-user documentation.
+# a book-like collection of end-user documentation.
#
# The Metron development community has chosen to do most documentation in README.md
# files, because they are easy to write and maintain, and located near the code they
@@ -48,7 +48,7 @@ METRON_SOURCE=`cd $(dirname $0); cd ../..; pwd`
EXCLUSION_LIST=(
'/site/'
'/site-book/'
- '/build_utils/'
+ '/dev-utilities/'
'/node_modules/'
'/\.github/'
)
@@ -330,4 +330,3 @@ else
echo " "
exit 0
fi
-
[14/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/opentaxii-conf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/opentaxii-conf.yml b/metron-deployment/roles/opentaxii/templates/opentaxii-conf.yml
deleted file mode 100644
index 2ce81c0..0000000
--- a/metron-deployment/roles/opentaxii/templates/opentaxii-conf.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-domain: "{{ opentaxii_domain }}"
-support_basic_auth: yes
-
-persistence_api:
- class: opentaxii.persistence.sqldb.SQLDatabaseAPI
- parameters:
- db_connection: "sqlite:///{{ opentaxii_data_db }}"
- create_tables: yes
-
-auth_api:
- class: opentaxii.auth.sqldb.SQLDatabaseAPI
- parameters:
- db_connection: "sqlite:///{{ opentaxii_auth_db }}"
- create_tables: yes
- secret: "{{ opentaxii_salt }}"
-
-logging:
- opentaxii: debug
- root: info
-
-hooks:
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/services.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/services.yml b/metron-deployment/roles/opentaxii/templates/services.yml
deleted file mode 100644
index 61a1d42..0000000
--- a/metron-deployment/roles/opentaxii/templates/services.yml
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-services:
- #
- # discovery: used by a TAXII Client to discover available TAXII Service
- #
- - id: discovery
- type: discovery
- address: /services/discovery
- description: Discovery service for Apache Metron
- advertised_services:
- - inbox
- - collection
- - poll
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
-
- #
- # inbox: used by a TAXII Client to push information to a TAXII Server
- #
- - id: inbox
- type: inbox
- address: /services/inbox
- description: Inbox for Apache Metron
- destination_collection_required: yes
- accept_all_content: yes
- supported_content:
- - urn:stix.mitre.org:xml:1.1.1
- authentication_required: no
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
-
- #
- # collection_management: used by a TAXII Client to request information about
- # available data collections or request a subscription.
- #
- - id: collection
- type: collection_management
- address: /services/collection
- description: Collection management service for Apache Metron
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
-
- #
- # poll: used by a TAXII Client to request information from a TAXII Server
- #
- - id: poll
- type: poll
- address: /services/poll
- description: Poll service for Apache Metron
- subscription_required: no
- authentication_required: no
- protocol_bindings:
- - urn:taxii.mitre.org:protocol:http:1.0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/README.md b/metron-deployment/roles/pcap_replay/README.md
deleted file mode 100644
index 01475f9..0000000
--- a/metron-deployment/roles/pcap_replay/README.md
+++ /dev/null
@@ -1,61 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-Pcap Replay
-===========
-
-This project enables packet capture data to be replayed through a network interface to simulate live network traffic. This can be used to support functional, performance, and load testing of Apache Metron.
-
-Getting Started
----------------
-
-To replay packet capture data, simply start the `pcap-replay` SysV service. To do this run the following command.
-
-```
-service pcap-replay start
-```
-
-All additional options accepted by `tcpreplay` can be passed to the service script to modify how the network data is replayed. For example, this makes it simple to control the amount and rate of data replayed during functional, performance and load testing.
-
-Example: Replay data at a rate of 10 mbps.
-
-```
-service pcap-replay start --mbps 10
-```
-
-Example: Replay data at a rate of 10 packets per second.
-
-```
-service pcap-replay start --pps 10
-```
-
-All nodes on the same subnet with their network interface set to promiscuous mode will then be able to capture the network traffic being replayed. To validate, simply run something like the following.
-
-```
-tcpdump -i eth1
-```
-
-Data
-----
-
-An example packet capture file has been installed at `/opt/pcap-replay/example.pcap`. By default, the network traffic contained within this file is continually replayed.
-
-To replay your own packet capture data, simply add any number of files containing `libpcap` formatted packet capture data to `/opt/pcap-replay`. The files must end with the `.pcap` extension. To pick up newly installed files, simply restart the service.
-
-```
-service pcap-replay restart
-```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/defaults/main.yml b/metron-deployment/roles/pcap_replay/defaults/main.yml
deleted file mode 100644
index 9b948a3..0000000
--- a/metron-deployment/roles/pcap_replay/defaults/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-pcap_replay_interface: eth0
-pcap_replay_home: /opt/pcap-replay
-tcpreplay_version: 4.1.1
-tcpreplay_prefix: /opt
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/meta/main.yml b/metron-deployment/roles/pcap_replay/meta/main.yml
deleted file mode 100644
index 841d185..0000000
--- a/metron-deployment/roles/pcap_replay/meta/main.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/dependencies.yml b/metron-deployment/roles/pcap_replay/tasks/dependencies.yml
deleted file mode 100644
index effe6c4..0000000
--- a/metron-deployment/roles/pcap_replay/tasks/dependencies.yml
+++ /dev/null
@@ -1,32 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install dependencies
- yum: name={{ item }}
- with_items:
- - "@Development tools"
- - libpcap
- - libpcap-devel
- - pcre
- - pcre-devel
- - zlib
- - zlib-devel
- - glib2-devel
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/main.yml b/metron-deployment/roles/pcap_replay/tasks/main.yml
deleted file mode 100644
index bdc2a0b..0000000
--- a/metron-deployment/roles/pcap_replay/tasks/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: tcpreplay.yml
-- include: service.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/tasks/service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/service.yml b/metron-deployment/roles/pcap_replay/tasks/service.yml
deleted file mode 100644
index 46b00c5..0000000
--- a/metron-deployment/roles/pcap_replay/tasks/service.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create pcap directory
- file: path={{ pcap_replay_home }} state=directory mode=0755
-
-- name: Install init.d service script
- template: src=pcap-replay dest=/etc/init.d/pcap-replay mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml b/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml
deleted file mode 100644
index e24dcf1..0000000
--- a/metron-deployment/roles/pcap_replay/tasks/tcpreplay.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download tcpreplay
- get_url:
- url: "https://github.com/appneta/tcpreplay/releases/download/v{{ tcpreplay_version }}/tcpreplay-{{ tcpreplay_version }}.tar.gz"
- dest: "/tmp/tcpreplay-{{ tcpreplay_version }}.tar.gz"
-
-- name: Extract tcpreplay tarball
- unarchive:
- src: "/tmp/tcpreplay-{{ tcpreplay_version }}.tar.gz"
- dest: /opt
- copy: no
- creates: "/opt/tcpreplay-{{ tcpreplay_version }}"
-
-- name: Compile and install tcpreplay
- shell: "{{ item }}"
- args:
- chdir: "/opt/tcpreplay-{{ tcpreplay_version }}"
- creates: "{{ tcpreplay_prefix }}/bin/tcpreplay"
- with_items:
- - "./configure --prefix={{ tcpreplay_prefix }}"
- - make
- - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pcap_replay/templates/pcap-replay
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pcap_replay/templates/pcap-replay b/metron-deployment/roles/pcap_replay/templates/pcap-replay
deleted file mode 100644
index 43ecc82..0000000
--- a/metron-deployment/roles/pcap_replay/templates/pcap-replay
+++ /dev/null
@@ -1,97 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# pcap replay daemon
-# chkconfig: 345 20 80
-# description: Replays packet capture data stored in libpcap format
-# processname: pcap-replay
-#
-
-DAEMON_PATH="{{ pcap_replay_home }}"
-PCAPIN=`ls $DAEMON_PATH/*.pcap 2> /dev/null`
-IFACE="{{ pcap_replay_interface }}"
-EXTRA_ARGS="${@:2}"
-NAME=pcap-replay
-DESC="Replay packet capture data"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-LOGFILE=/var/log/$NAME.log
-
-case "$1" in
- start)
- printf "%-50s" "Starting $NAME..."
-
- # ensure that a pcap file exists to replay
- if [ -z "$PCAPIN" ]; then
- printf "%s: %s\n" "Fail: No pcap files found at " $DAEMON_PATH
- else
- # kick-off the daemon
- cd $DAEMON_PATH
- DAEMON="{{ tcpreplay_prefix }}/bin/tcpreplay"
- DAEMONOPTS="--intf1=$IFACE --loop=0 $EXTRA_ARGS $PCAPIN"
- PID=`$DAEMON $DAEMONOPTS > $LOGFILE 2>&1 & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- fi
- ;;
-
- status)
- printf "%-50s" "Checking $NAME..."
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- echo "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- stop)
- printf "%-50s" "Stopping $NAME"
- PID=`cat $PIDFILE`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- while sleep 1
- echo -n "."
- kill -0 $PID >/dev/null 2>&1
- do
- kill $PID
- done
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/defaults/main.yml b/metron-deployment/roles/pycapa/defaults/main.yml
deleted file mode 100644
index edcf980..0000000
--- a/metron-deployment/roles/pycapa/defaults/main.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-pycapa_home: /usr/local/pycapa
-pycapa_venv: pycapa-venv
-pycapa_bin: "{{ pycapa_home }}/{{ pycapa_venv }}/bin"
-pycapa_log: /var/log/pycapa.log
-pycapa_topic: pcap
-pycapa_sniff_interface: "{{ sniff_interface }}"
-python27_home: /opt/rh/python27/root
-
-install_pycapa_service: True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/meta/main.yml b/metron-deployment/roles/pycapa/meta/main.yml
deleted file mode 100644
index a5b54b7..0000000
--- a/metron-deployment/roles/pycapa/meta/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - ambari_gather_facts
- - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/tasks/dependencies.yml b/metron-deployment/roles/pycapa/tasks/dependencies.yml
deleted file mode 100644
index 2fba127..0000000
--- a/metron-deployment/roles/pycapa/tasks/dependencies.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install yum repositories
- yum: name={{ item }} update_cache=yes
- with_items:
- - epel-release
- - centos-release-scl
-
-- name: Install dependencies
- yum: name={{ item }}
- with_items:
- - "@Development tools"
- - python27
- - python27-scldevel
- - python27-python-virtualenv
- - libpcap-devel
- - libselinux-python
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/tasks/main.yml b/metron-deployment/roles/pycapa/tasks/main.yml
deleted file mode 100644
index d2367c2..0000000
--- a/metron-deployment/roles/pycapa/tasks/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: pycapa.yml
-- include: pycapa-service.yml
- when: install_pycapa_service
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/tasks/pycapa-service.yml b/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
deleted file mode 100644
index 017cc66..0000000
--- a/metron-deployment/roles/pycapa/tasks/pycapa-service.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Turn on promiscuous mode for {{ pycapa_sniff_interface }}
- shell: "ip link set {{ pycapa_sniff_interface }} promisc on"
-
-- name: Install service script
- template: src=pycapa dest=/etc/init.d/pycapa mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/tasks/pycapa.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/tasks/pycapa.yml b/metron-deployment/roles/pycapa/tasks/pycapa.yml
deleted file mode 100644
index 87e964e..0000000
--- a/metron-deployment/roles/pycapa/tasks/pycapa.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create install directory
- file: path={{ pycapa_home }} state=directory mode=0755
-
-- name: Create virtual environment
- shell: "{{ python27_home }}/usr/bin/virtualenv {{ pycapa_venv }}"
- args:
- chdir: "{{ pycapa_home }}"
- creates: "{{ pycapa_home }}/{{ pycapa_venv }}"
- environment:
- LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
-
-- name: Distribute pycapa
- copy: src=../../../metron-sensors/pycapa dest={{ pycapa_home }} mode=0755
-
-- name: Build pycapa
- shell: "{{ item }}"
- args:
- chdir: "{{ pycapa_home }}/pycapa"
- environment:
- LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
- with_items:
- - "{{ pycapa_bin }}/pip install -r requirements.txt"
- - "{{ pycapa_bin }}/python setup.py install"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/pycapa/templates/pycapa
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/pycapa/templates/pycapa b/metron-deployment/roles/pycapa/templates/pycapa
deleted file mode 100644
index ce87477..0000000
--- a/metron-deployment/roles/pycapa/templates/pycapa
+++ /dev/null
@@ -1,115 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# metron pycapa service
-# chkconfig: 345 20 80
-# description: Metron Pycapa Packet Capture Daemon
-# processname: pycapa
-#
-NAME=pycapa
-DESC="Pycapa - Apache Metron Packet Capture"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-LOGFILE="{{ pycapa_log }}"
-EXTRA_ARGS="${@:2}"
-DAEMON_PATH="{{ pycapa_home }}"
-
-export LD_LIBRARY_PATH={{ python27_home }}/usr/lib64
-
-case "$1" in
-
- ##############################################################################
- # start
- #
- start)
- printf "%-50s" "Starting $NAME..."
-
- # setup virtual environment
- cd $DAEMON_PATH
- . {{ pycapa_bin }}/activate
-
- # kick-off the daemon
- DAEMON_PATH="{{ pycapa_home }}"
- DAEMON="{{ pycapa_bin }}/pycapa"
- DAEMONOPTS+=" --producer "
- DAEMONOPTS+=" --kafka {{ kafka_broker_url }}"
- DAEMONOPTS+=" --topic {{ pycapa_topic }}"
- DAEMONOPTS+=" --interface {{ pycapa_sniff_interface }}"
- DAEMONOPTS+=" $EXTRA_ARGS"
-
- PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- ;;
-
- ##############################################################################
- # status
- #
- status)
- printf "%-50s" "Checking $NAME..."
- . {{ pycapa_bin }}/activate
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- printf "%s\n" "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- ##############################################################################
- # stop
- #
- stop)
- printf "%-50s" "Stopping $NAME"
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- while sleep 1
- echo -n "."
- kill -0 $PID >/dev/null 2>&1
- do
- kill $PID
- done
-
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- ##############################################################################
- # restart
- #
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/python-pip/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/python-pip/tasks/main.yml b/metron-deployment/roles/python-pip/tasks/main.yml
deleted file mode 100644
index 809aca4..0000000
--- a/metron-deployment/roles/python-pip/tasks/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install python-pip
- yum:
- name: python-pip
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/README.md b/metron-deployment/roles/sensor-stubs/README.md
deleted file mode 100644
index 3e99cfc..0000000
--- a/metron-deployment/roles/sensor-stubs/README.md
+++ /dev/null
@@ -1,99 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Sensor Stubs
-
-A service has been created to simulate the behavior of a sensor by sending canned telemetry data to a Kafka topic. These "Sensor Stubs" consume fewer resources than the actual sensor that they replace.
-
-### (Q) How do the sensor stubs work?
-
-The stubs are installed with a set of canned data for each sensor type; Bro, Snort and YAF. A subset of this canned data is randomly selected and sent to the Kafka topic in batches. The timestamp of each message is updated to match current system time.
-
-### (Q) How do I configure the message rate?
-
-The number of telemetry messages sent in each batch, along with the time delay between batches is configurable. Before installation, these values can be configured by redefining `sensor_stubs_delay` and `sensor_stubs_count`. The values can also be configured by altering the deployed system service script at `/etc/init.d/sensor-stubs`.
-
-### (Q) How do I install the sensor stubs?
-
-Using the default playbooks, this role can be installed by using the Ansible tag `sensor-stubs`. This service is installed on the same hosts where the sensors would be; defined by the `sensors` host group.
-
-The defaults for the "Full Dev" environment have been changed so that the Sensor Stubs are installed by default, rather than the sensors themselves. The Amazon EC2 environment continues to install the original sensors by default.
-
-### (Q) How do I use the sensor stubs?
-
-Start all sensor stubs. The output includes the PID for each running sensor stub.
-```
-$ service sensor-stubs start
-Starting sensor-stubs...
- bro: Ok [26505]
- yaf: Ok [26507]
- snort: Ok [26509]
-```
-
-Check the status of each sensor stub.
-```
-$ service sensor-stubs status
-Checking sensor-stubs...
- bro: Running [26505]
- yaf: Running [26507]
- snort: Running [26509]
-```
-
-Stop all sensor stubs.
-```
-$ service sensor-stubs stop
-Stopping sensor-stubs...
-.. bro: Ok [26505]
-.. yaf: Ok [26507]
-.. snort: Ok [26509]
-```
-
-Check the status. All sensor stubs should be stopped.
-```
-$ service sensor-stubs status
-Checking sensor-stubs...
- bro: Not running
- yaf: Not running
- snort: Not running
-```
-
-Start only the Bro sensor stub.
-```
-$ service sensor-stubs start bro
-Starting sensor-stubs...
- bro: OK [11616]
-```
-
-Stop the Bro sensor stub.
-```
-$ service sensor-stubs stop bro
-Stopping sensor-stubs...
-.. bro: Ok [11616]
-```
-
-### (Q) How do I install the original sensors?
-
-The default behavior can be changed by skipping the `sensor-stubs` flag and including the `sensors` flag. For example, to deploy "Full Dev" with the original sensors run the following command.
-
-```
-cd metron-deployment/vagrant/full-dev-platform
-vagrant --ansible-skip-tags="sensor-stubs,solr" up
-```
-
-### (Q) Where does the mock data come from?
-
-The data produced by the sensor stubs was generated by running the sensors against the example pcap file that is distributed with Metron. This ensures that the data produced by the sensor stubs is similar to the data produced when using the actual sensors.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/defaults/main.yml b/metron-deployment/roles/sensor-stubs/defaults/main.yml
deleted file mode 100644
index e8efb9e..0000000
--- a/metron-deployment/roles/sensor-stubs/defaults/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-sensor_stubs_home: /opt/sensor-stubs
-sensor_stubs_bin: "{{ sensor_stubs_home }}/bin"
-sensor_stubs_data: "{{ sensor_stubs_home }}/data"
-sensor_stubs_log: /var/log/sensor-stubs.log
-
-sensor_stubs_delay: 2
-sensor_stubs_count: 10
-kafka_home: /usr/hdp/current/kafka-broker
\ No newline at end of file
[30/50] [abbrv] metron git commit: METRON-1432 JDK Install Fails on
Ubuntu Development Environment (nickwallen) closes apache/metron#913
Posted by rm...@apache.org.
METRON-1432 JDK Install Fails on Ubuntu Development Environment (nickwallen) closes apache/metron#913
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/a285b83e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/a285b83e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/a285b83e
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: a285b83ee0153cecf24132b4aed70250040a83b0
Parents: 9be0d85
Author: nickwallen <ni...@nickallen.org>
Authored: Tue Jan 30 09:31:53 2018 -0500
Committer: nickallen <ni...@apache.org>
Committed: Tue Jan 30 09:31:53 2018 -0500
----------------------------------------------------------------------
.../roles/java_jdk/tasks/install_jdk_centos.yml | 34 ++++++++++++++++++++
.../roles/java_jdk/tasks/install_jdk_ubuntu.yml | 31 ++++++++++++++++++
.../ansible/roles/java_jdk/tasks/main.yml | 22 ++++---------
3 files changed, 71 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/a285b83e/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_centos.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_centos.yml b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_centos.yml
new file mode 100644
index 0000000..999b9c1
--- /dev/null
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_centos.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Check for java at "{{ java_home }}"
+ stat: path="{{ java_home }}"
+ register: jdk_dir
+
+- name: Alternatives link for java
+ alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
+ with_items:
+ - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
+ - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
+ when: jdk_dir.stat.exists
+
+- name: Install openjdk
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - java-1.8.0-openjdk-devel
+ when: not jdk_dir.stat.exists
http://git-wip-us.apache.org/repos/asf/metron/blob/a285b83e/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
new file mode 100644
index 0000000..8337b81
--- /dev/null
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/install_jdk_ubuntu.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Check for java at "{{ java_home }}"
+ stat: path="{{ java_home }}"
+ register: jdk_dir
+
+- name: Install openjdk repository
+ shell: add-apt-repository ppa:openjdk-r/ppa
+ when: not jdk_dir.stat.exists
+
+- name: Update package cache
+ apt: update_cache=yes
+
+- name: Install openjdk
+ apt: name=openjdk-8-jdk
+ when: not jdk_dir.stat.exists
http://git-wip-us.apache.org/repos/asf/metron/blob/a285b83e/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/main.yml b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
index 999b9c1..946da25 100644
--- a/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
@@ -15,20 +15,10 @@
# limitations under the License.
#
---
-- name: Check for java at "{{ java_home }}"
- stat: path="{{ java_home }}"
- register: jdk_dir
+- include: install_jdk_centos.yml
+ static: no
+ when: ansible_distribution == "CentOS"
-- name: Alternatives link for java
- alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
- with_items:
- - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
- - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
- when: jdk_dir.stat.exists
-
-- name: Install openjdk
- yum: name={{item}}
- with_items:
- - java-1.8.0-openjdk
- - java-1.8.0-openjdk-devel
- when: not jdk_dir.stat.exists
+- include: install_jdk_ubuntu.yml
+ static: no
+ when: ansible_distribution == "Ubuntu"
[29/50] [abbrv] metron git commit: METRON-1431 Add REGEXP_REPLACE
function to Stellar (havran via ottobackwards) closes apache/metron#912
Posted by rm...@apache.org.
METRON-1431 Add REGEXP_REPLACE function to Stellar (havran via ottobackwards) closes apache/metron#912
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9be0d858
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9be0d858
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9be0d858
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 9be0d858452ebc836e812d665c55a5481fa6fea2
Parents: 0c6aad8
Author: havran <ha...@gmail.com>
Authored: Mon Jan 29 16:44:33 2018 -0500
Committer: otto <ot...@apache.org>
Committed: Mon Jan 29 16:44:33 2018 -0500
----------------------------------------------------------------------
metron-stellar/stellar-common/README.md | 9 +++++
.../stellar/dsl/functions/RegExFunctions.java | 36 ++++++++++++++++++++
.../dsl/functions/RegExFunctionsTest.java | 19 +++++++++++
3 files changed, 64 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/9be0d858/metron-stellar/stellar-common/README.md
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/README.md b/metron-stellar/stellar-common/README.md
index 076f250..98aea77 100644
--- a/metron-stellar/stellar-common/README.md
+++ b/metron-stellar/stellar-common/README.md
@@ -232,6 +232,7 @@ Where:
| [ `REDUCE`](#reduce) |
| [ `REGEXP_MATCH`](#regexp_match) |
| [ `REGEXP_GROUP_VAL`](#regexp_group_val) |
+| [ `REGEXP_REPLACE`](#regexp_replace) |
| [ `ROUND`](#round) |
| [ `SAMPLE_ADD`](../../metron-analytics/metron-statistics#sample_add) |
| [ `SAMPLE_GET`](../../metron-analytics/metron-statistics#sample_get) |
@@ -879,6 +880,14 @@ Where:
* group - The integer that selects what group to select, starting at 1
* Returns: The value of the group, or null if not matched or no group at index.
+### `REGEXP_REPLACE`
+ * Description: Replace all occurences of the regex pattern within the string by value
+ * Input:
+ * string - The input string
+ * pattern - The proposed regex pattern
+ * value - The value to replace the regex pattern
+ * Returns: The modified input string with replaced values
+
### `ROUND`
* Description: Rounds a number to the nearest integer. This is half-up rounding.
* Input:
http://git-wip-us.apache.org/repos/asf/metron/blob/9be0d858/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
index a1ea229..ddc8860 100644
--- a/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
+++ b/metron-stellar/stellar-common/src/main/java/org/apache/metron/stellar/dsl/functions/RegExFunctions.java
@@ -21,6 +21,7 @@ package org.apache.metron.stellar.dsl.functions;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.apache.commons.lang3.StringUtils;
import org.apache.metron.stellar.common.utils.ConversionUtils;
import org.apache.metron.stellar.common.utils.PatternCache;
import org.apache.metron.stellar.dsl.BaseStellarFunction;
@@ -100,4 +101,39 @@ public class RegExFunctions {
return matcher.group(groupNumber);
}
}
+
+ @Stellar(name = "REGEXP_REPLACE",
+ description = "Replace all occurences of the regex pattern within the string by value",
+ params = {
+ "string - The input string",
+ "pattern - The regex pattern to be replaced. Special characters must be escaped (e.g. \\\\d)",
+ "value - The value to replace the regex pattern"
+ },
+ returns = "The modified input string with replaced values")
+ public static class RegexpReplace extends BaseStellarFunction {
+
+ @Override
+ public Object apply(List<Object> list) {
+ if (list.size() != 3) {
+ throw new IllegalStateException(
+ "REGEXP_REPLACE expects three args: [string, pattern, value]"
+ + " where pattern is a regexp pattern");
+ }
+ String str = (String) list.get(0);
+ String stringPattern = (String) list.get(1);
+ String value = (String) list.get(2);
+
+ if (StringUtils.isEmpty(str)) {
+ return null;
+ }
+
+ if (StringUtils.isEmpty(stringPattern) || StringUtils.isEmpty(value)) {
+ return str;
+ }
+
+ Pattern pattern = PatternCache.INSTANCE.getPattern(stringPattern);
+ Matcher matcher = pattern.matcher(str);
+ return matcher.replaceAll(value);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9be0d858/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RegExFunctionsTest.java
----------------------------------------------------------------------
diff --git a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RegExFunctionsTest.java b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RegExFunctionsTest.java
index 2aefb67..f0b579f 100644
--- a/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RegExFunctionsTest.java
+++ b/metron-stellar/stellar-common/src/test/java/org/apache/metron/stellar/dsl/functions/RegExFunctionsTest.java
@@ -68,4 +68,23 @@ public class RegExFunctionsTest {
Assert.assertTrue("Did not fail on wrong number of parameters",false);
}
}
+
+ @Test
+ public void testRegExReplace() throws Exception {
+ final Map<String, String> variableMap = new HashMap<String, String>() {{
+ put("numbers", "12345");
+ put("numberPattern", "\\d(\\d)(\\d).*");
+ put("letters", "abcde");
+ put("empty", "");
+ }};
+
+ Assert.assertTrue(runPredicate("REGEXP_REPLACE(empty, numberPattern, letters) == null", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("REGEXP_REPLACE(numbers, empty, empty) == numbers", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("REGEXP_REPLACE(numbers, empty, letters) == numbers", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("REGEXP_REPLACE(numbers, numberPattern, empty) == numbers", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("REGEXP_REPLACE(numbers, numberPattern, letters) == letters", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ Assert.assertTrue(runPredicate("REGEXP_REPLACE(letters, numberPattern, numbers) == letters", new DefaultVariableResolver(v -> variableMap.get(v),v -> variableMap.containsKey(v))));
+ }
+
+
}
[17/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst
new file mode 100644
index 0000000..42a2ca7
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postinst
@@ -0,0 +1,26 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed AFTER installation of the 'metron-config' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+# install the service script
+cp -f /usr/metron/${FULL_VERSION}/bin/metron-management-ui /etc/init.d/
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm
new file mode 100644
index 0000000..f9f4bcd
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/postrm
@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed AFTER removal of the 'metron-management' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+rm -f /etc/init.d/metron-management-ui
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst
new file mode 100644
index 0000000..3ddedc9
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/preinst
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed BEFORE installation of the 'metron-config' package
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+if [ -f "/etc/init.d/metron-management-ui"]; then
+ # if service already exists, stop it before upgrading
+ /etc/init.d/metron-management-ui stop
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm
new file mode 100644
index 0000000..5b2ad74
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron-config/prerm
@@ -0,0 +1,30 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this script is executed BEFORE removal of the 'metron-management' package.
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+# if service exists, stop it
+if [ -f "/etc/init.d/metron-management-ui"]; then
+ /etc/init.d/metron-management-ui stop
+fi
+
+exit 0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog b/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog
new file mode 100755
index 0000000..cfbf6af
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/changelog
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this file is added to ALL metron packages
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+$PACKAGE ($FULL_VERSION) $DISTRIBUTION; urgency=medium
+
+ * Initial release. (Closes: METRON-1351)
+
+ -- Apache Metron <de...@metron.apache.org> Wed, 13 Dec 2017 21:19:45 +0000
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/control
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/control b/metron-deployment/packaging/docker/deb-docker/debian/metron/control
new file mode 100755
index 0000000..d7611f9
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/control
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this file is added to ALL metron packages
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+Source: metron
+Section: misc
+Priority: extra
+Maintainer: Apache Metron <de...@metron.apache.org>
+Homepage: https://metron.apache.org/
+Package: $PACKAGE
+Architecture: all
+Version: $FULL_VERSION
+Depends:
+Description: Apache Metron
+ Apache Metron provides a scalable advanced security analytics framework.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright b/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright
new file mode 100755
index 0000000..d7f20c1
--- /dev/null
+++ b/metron-deployment/packaging/docker/deb-docker/debian/metron/copyright
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# this file is added to ALL metron packages
+# comments are stripped from this file before packaging
+# environment variable substitution is performed on this file before packaging
+#
+
+Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: $PACKAGE
+Upstream-Contact: Apache Metron <de...@metron.apache.org>
+
+Files: *
+License: ASL-2
+Copyright:
+Apache Metron
+Copyright 2015-2016 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+This product includes software developed by Cisco Systems (http://www.cisco.com)
+Copyright (c) 2014 Cisco Systems.
+
+This product includes software developed by Chef Software (https://www.chef.io)
+Copyright (c) 2012-2015, Chef Software, Inc.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/deb-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/deb-docker/pom.xml b/metron-deployment/packaging/docker/deb-docker/pom.xml
index 5770e40..7fbe47b 100644
--- a/metron-deployment/packaging/docker/deb-docker/pom.xml
+++ b/metron-deployment/packaging/docker/deb-docker/pom.xml
@@ -49,6 +49,12 @@
<include>**/*</include>
</includes>
</fileset>
+ <fileset>
+ <directory>.npm</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </fileset>
</filesets>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/packaging/docker/rpm-docker/pom.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/docker/rpm-docker/pom.xml b/metron-deployment/packaging/docker/rpm-docker/pom.xml
index 0b7218d..4ed2edd 100644
--- a/metron-deployment/packaging/docker/rpm-docker/pom.xml
+++ b/metron-deployment/packaging/docker/rpm-docker/pom.xml
@@ -85,6 +85,12 @@
<directory>SRPMS</directory>
<followSymlinks>false</followSymlinks>
</fileset>
+ <fileset>
+ <directory>.npm</directory>
+ <includes>
+ <include>**/*</include>
+ </includes>
+ </fileset>
</filesets>
</configuration>
</plugin>
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/ambari_install.yml b/metron-deployment/playbooks/ambari_install.yml
deleted file mode 100644
index c0cfef3..0000000
--- a/metron-deployment/playbooks/ambari_install.yml
+++ /dev/null
@@ -1,61 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: ec2
- become: true
- tasks:
- - include_vars: ../amazon-ec2/conf/defaults.yml
- tags:
- - ec2
-
-- hosts: packer
- become: true
- tasks:
- - include_vars: ../inventory/full-dev-platform/group_vars/all
- tags:
- - packer
-
-- hosts: ambari_*
- become: true
- roles:
- - role: ambari_common
- tags:
- - ambari-prereqs
- - hdp-install
-
-- hosts: ambari_master
- become: true
- roles:
- - role: ambari_master
- tags:
- - ambari-server
- - hdp-install
-
-- hosts: ambari_slave
- become: true
- roles:
- - role: ambari_slave
- tags:
- - ambari-agent
- - hdp-install
-
-- hosts: metron
- become: true
- roles:
- - role: metron-rpms
- tags:
- - metron-deploy
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/docker_probe_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/docker_probe_install.yml b/metron-deployment/playbooks/docker_probe_install.yml
deleted file mode 100644
index 7be779a..0000000
--- a/metron-deployment/playbooks/docker_probe_install.yml
+++ /dev/null
@@ -1,62 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# sensors
-#
-- hosts: localhost
- tasks:
- - name: add container to inventory
- add_host:
- name: amb-server
- ansible_connection: docker
- groups: sensors
- changed_when: false
- tags: add-host
-
-- hosts: sensors
- vars:
- metron_version: 0.4.3
- metron_directory: /usr/metron/{{ metron_version }}
- bro_version: "2.5.2"
- fixbuf_version: "1.7.1"
- yaf_version: "2.8.0"
- daq_version: "2.0.6-1"
- pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
- pycapa_home: "/opt/pycapa"
- snort_version: "2.9.8.0-1"
- snort_alert_csv_path: "/var/log/snort/alert.csv"
- threat_intel_bulk_load: False
- sensor_test_mode: True
- install_pycapa: False
- install_bro: True
- install_snort: True
- install_yaf: True
- install_pcap_replay: True
- sniff_interface: eth0
- pcap_replay_interface: "{{ sniff_interface }}"
- pcapservice_port: 8081
- kafka_broker_url: amb4.service.consul:6667
- connection: docker
- roles:
- - { role: bro, tags: ['bro'] }
- - { role: snort, tags: ['snort'] }
- - { role: yaf, tags: ['yaf'] }
- - { role: pcap_replay, tags: ['pcap-replay'] }
- - { role: sensor-test-mode, tags: ['sensor-test-mode'] }
- tags:
- - sensors
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_build.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_build.yml b/metron-deployment/playbooks/metron_build.yml
deleted file mode 100644
index 83eb352..0000000
--- a/metron-deployment/playbooks/metron_build.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: all
- become: false
- roles:
- - role: metron-builder
- tags:
- - build
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_full_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_full_install.yml b/metron-deployment/playbooks/metron_full_install.yml
deleted file mode 100644
index 670f5c8..0000000
--- a/metron-deployment/playbooks/metron_full_install.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: all
- pre_tasks:
- - name: Verify Ansible Version
- fail: msg="Metron Requires Ansible 2.0.0.2 or 2.2.2.0, current version is {{ ansible_version }}"
- when: "ansible_version.full | version_compare('2.2.2.0', '!=') and ansible_version.full | version_compare('2.0.0.2', '!=')"
-
-- include: metron_build.yml
- tags:
- - build
-
-- include: ambari_install.yml
- tags:
- - ambari
-
-- include: metron_install.yml
- tags:
- - metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/playbooks/metron_install.yml b/metron-deployment/playbooks/metron_install.yml
deleted file mode 100644
index bed615e..0000000
--- a/metron-deployment/playbooks/metron_install.yml
+++ /dev/null
@@ -1,102 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- hosts: ec2
- become: true
- tasks:
- - include_vars: ../amazon-ec2/conf/defaults.yml
- tags:
- - ec2
-
-- hosts: packer
- become: true
- tasks:
- - include_vars: ../inventory/full-dev-platform/group_vars/all
- tags:
- - packer
-
-#
-# start installation of components in Ambari
-#
-- hosts: ambari_master
- become: true
- roles:
- - role: ambari_config
- tags:
- - hdp-install
- - hdp-deploy
-
-- hosts: ambari_master
- become: true
- roles:
- - role: load_web_templates
- tags:
- - load_templates
-
-- hosts: pcap_server
- become: true
- roles:
- - role: metron_pcapservice
- tags:
- - pcap-service
-
-#
-# sensors
-#
-- hosts: sensors
- become: true
- roles:
- - { role: ambari_gather_facts, tags: ['always'] }
- - { role: tap_interface, tags: ['tap'] }
- - { role: pycapa, tags: ['pycapa'] }
- - { role: bro, tags: ['bro'] }
- - { role: snort, tags: ['snort'] }
- - { role: yaf, tags: ['yaf'] }
- - { role: pcap_replay, tags: ['pcap-replay'] }
- - { role: sensor-test-mode, tags: ['sensor-test-mode'] }
- tags:
- - sensors
-
-#
-# sensor stubs - rather than deploying the sensors, deploy ligher weight
-# stubs that mimic the behavior of the real sensors
-#
-- hosts: sensors
- become: true
- roles:
- - { role: ambari_gather_facts }
- - { role: sensor-stubs }
- tags:
- - sensor-stubs
-
-#
-# monitor and start metron services with monit
-#
-- hosts: monit
- become: true
- roles:
- - { role: ambari_gather_facts, tags: ['always'] }
- - { role: monit, tags: ['monit'] }
- - { role: monit-start, tags: ['start'] }
-
-#
-# deployment report
-#
-- hosts: monit
- become: false
- roles:
- - { role: deployment-report, tags: ['report'] }
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/README.md b/metron-deployment/roles/README.md
deleted file mode 100644
index 1cb5878..0000000
--- a/metron-deployment/roles/README.md
+++ /dev/null
@@ -1,24 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Ansible Roles
-
-- Monit
-- OpenTaxii
-- Pcap Replay
-- Sensor Stubs
-- Sensor Test Mode
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/defaults/main.yml b/metron-deployment/roles/ambari_common/defaults/main.yml
deleted file mode 100644
index 2c9389c..0000000
--- a/metron-deployment/roles/ambari_common/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hadoop_logrotate_frequency: daily
-hadoop_logrotate_retention: 30
-rhel_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/meta/main.yml b/metron-deployment/roles/ambari_common/meta/main.yml
deleted file mode 100644
index d7e46d6..0000000
--- a/metron-deployment/roles/ambari_common/meta/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - libselinux-python
- - epel
- - ntp
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/tasks/main.yml b/metron-deployment/roles/ambari_common/tasks/main.yml
deleted file mode 100644
index 011c05f..0000000
--- a/metron-deployment/roles/ambari_common/tasks/main.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Check OS Version
- fail: msg="Ambari HDP deployment supports CentOS 6 only."
- when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
-
-- name: Ensure iptables is stopped and is not running at boot time.
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
-
-#
-# ambari uses socket.getfqdn() to find the hostname. with 'localhost.localdomain'
-# in '/etc/hosts' this function will report the hostname as 'localhost.localdomain'
-# rather than 'node1' as would be expected. other functions like socket.gethostname()
-# will always return 'node1' as expected. ambari needs to see 'node1' to be able to
-# communicate between the master and agents.
-
-- name: Remove ipv4 'localhost.localdomain' from /etc/hosts
- lineinfile: dest=/etc/hosts state=absent regexp="^127.0.0.1(.*)localdomain(.*)$"
-
-- name: Remove ipv6 'localhost.localdomain' from /etc/hosts
- lineinfile: dest=/etc/hosts state=absent regexp="^::1(.*)localdomain(.*)$"
-
-- name: Add localhost to /etc/hosts
- lineinfile: dest=/etc/hosts line="127.0.0.1 localhost"
-
-- name: Download Ambari repo
- get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
-
-- name: Create Logrotate Script for Hadoop Services
- template:
- src: "metron-hadoop-logrotate.yml"
- dest: "/etc/logrotate.d/metron-ambari"
- mode: 0644
-
-- name: Install Nodejs repository
- shell: curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
- args:
- warn: false
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml b/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
deleted file mode 100644
index d95c10b..0000000
--- a/metron-deployment/roles/ambari_common/templates/metron-hadoop-logrotate.yml
+++ /dev/null
@@ -1,149 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#Hadoop HDFS Logs
-/var/log/hadoop/hdfs/*.log* {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hadoop/hdfs/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hadoop/hdfs/*.audit {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Hadoop Yarn Logs
-/var/log/hadoop/yarn/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Hadoop Mapreduce Logs
-/var/log/hadoop/mapreduce/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Storm Logs
-/var/log/storm/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/storm/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Kafka Logs
-/var/log/kafka/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/kafka/*.err {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#HBase Logs
-/var/log/hbase/*.log* {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hbase/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/hbase/*.audit {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-#Zookeeper Logs
-/var/log/zookeeper/*.log {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
-/var/log/zookeeper/*.out {
- {{ hadoop_logrotate_frequency }}
- rotate {{ hadoop_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/defaults/main.yml b/metron-deployment/roles/ambari_config/defaults/main.yml
deleted file mode 100644
index e0de145..0000000
--- a/metron-deployment/roles/ambari_config/defaults/main.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-zookeeper_data_dir: /hadoop/zookeeper
-namenode_checkpoint_dir: /hadoop/hdfs/namesecondary
-namenode_name_dir: /hadoop/hdfs/namenode
-datanode_data_dir: /hadoop/hdfs/data
-journalnode_edits_dir: /hadoop/hdfs/journalnode
-jhs_recovery_store_ldb_path: /hadoop/mapreduce/jhs
-nodemanager_local_dirs: /hadoop/yarn/local
-timeline_ldb_store_path: /hadoop/yarn/timeline
-timeline_ldb_state_path: /hadoop/yarn/timeline
-nodemanager_log_dirs: /hadoop/yarn/log
-storm_local_dir: /hadoop/storm
-kafka_log_dirs: /kafka-log
-cluster_type: small_cluster
-nodemanager_mem_mb : 4096
-mapred_map_java_opts : -Xmx1024m
-mapred_reduce_java_opts : -Xmx1024m
-mapred_map_mem_mb : 1229
-mapred_reduce_mem_mb : 1229
-topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf'
-hdp_stack: "2.5"
-elasticsearch_network_interface: _site_
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/meta/main.yml b/metron-deployment/roles/ambari_config/meta/main.yml
deleted file mode 100644
index 8f65a28..0000000
--- a/metron-deployment/roles/ambari_config/meta/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - epel
- - python-pip
- - httplib2
- - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/tasks/main.yml b/metron-deployment/roles/ambari_config/tasks/main.yml
deleted file mode 100644
index 6588d4a..0000000
--- a/metron-deployment/roles/ambari_config/tasks/main.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include_vars: "{{ cluster_type }}.yml"
-
-- name: Install python-requests
- yum:
- name: python-requests
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }}
- wait_for :
- host: "{{ ambari_host }}"
- port: "{{ ambari_port }}"
- timeout: 600
-
-- name: Deploy cluster with Ambari; http://{{ ambari_host }}:{{ ambari_port }}
- ambari_cluster_state:
- host: "{{ ambari_host }}"
- port: "{{ ambari_port }}"
- username: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- cluster_name: "{{ cluster_name }}"
- cluster_state: present
- blueprint_name: "{{ blueprint_name }}"
- configurations: "{{ configurations }}"
- wait_for_complete: True
- blueprint_var: "{{ blueprint }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
deleted file mode 100644
index 6a60902..0000000
--- a/metron-deployment/roles/ambari_config/vars/single_node_vm.yml
+++ /dev/null
@@ -1,135 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-# vars file for single_node_vm blueprint
-
-hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
-hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, HDFS_CLIENT, NODEMANAGER, YARN_CLIENT, MAPREDUCE2_CLIENT]
-spark_master: [SPARK_JOBHISTORYSERVER]
-spark_slave: [SPARK_CLIENT]
-storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
-storm_slave: [SUPERVISOR]
-kafka_broker: [KAFKA_BROKER]
-zookeeper_master: [ZOOKEEPER_SERVER]
-zookeeper_slave: [ZOOKEEPER_CLIENT]
-hbase_master: [HBASE_MASTER, HBASE_CLIENT]
-hbase_slave: [HBASE_REGIONSERVER]
-es_master: [ES_MASTER]
-kibana_master: [KIBANA_MASTER]
-metron_indexing: [METRON_INDEXING]
-metron_profiler: [METRON_PROFILER]
-metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
-metron_parsers : [METRON_PARSERS]
-metron_rest: [METRON_REST]
-metron_management_ui: [METRON_MANAGEMENT_UI]
-metron_alerts_ui: [METRON_ALERTS_UI]
-
-metron_components: >
- {{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) |
- union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(kibana_master) | union(metron_indexing) | union(metron_profiler) |
- union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(es_master) }}
-
-cluster_name: "metron_cluster"
-blueprint_name: "metron_blueprint"
-
-configurations:
- - zoo.cfg:
- dataDir: '{{ zookeeper_data_dir }}'
- - hadoop-env:
- hadoop_heapsize: 1024
- namenode_heapsize: 2048
- dtnode_heapsize: 512
- namenode_opt_permsize: 128m
- - hbase-env:
- hbase_regionserver_heapsize: 512
- hbase_master_heapsize: 512
- hbase_regionserver_xmn_max: 512
- - hdfs-site:
- dfs.replication: 1
- dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir }}'
- dfs.namenode.name.dir: '{{ namenode_name_dir }}'
- dfs.datanode.data.dir: '{{ datanode_data_dir }}'
- dfs.journalnode.edits.dir: '{{ journalnode_edits_dir }}'
- - yarn-env:
- nodemanager_heapsize: 512
- yarn_heapsize: 512
- apptimelineserver_heapsize : 512
- resourcemanager_heapsize: 1024
- - mapred-env:
- jobhistory_heapsize: 256
- - mapred-site:
- mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path }}'
- mapreduce.map.java.opts : '{{ mapred_map_java_opts }}'
- mapreduce.reduce.java.opts : '{{ mapred_reduce_java_opts }}'
- mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
- mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
- - yarn-site:
- yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs }}'
- yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path }}'
- yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path }}'
- yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs }}'
- yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
- - storm-site:
- supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
- storm.local.dir: '{{ storm_local_dir }}'
- topology.classpath: '{{ topology_classpath }}'
- - kafka-env:
- content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
- - kafka-broker:
- log.dirs: '{{ kafka_log_dirs }}'
- delete.topic.enable: "true"
- - metron-rest-env:
- metron_spring_profiles_active: "dev"
- - metron-parsers-env:
- parsers: "bro,snort"
- - elastic-site:
- index_number_of_shards: 1
- index_number_of_replicas: 0
- zen_discovery_ping_unicast_hosts: "[ {{ groups.search | join(', ') }} ]"
- gateway_recover_after_data_nodes: 1
- network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"
- masters_also_are_datanodes: "1"
-
-required_configurations:
- - metron-env:
- storm_rest_addr: "http://{{ groups.ambari_slave[0] }}:8744"
- es_hosts: "{{ groups.search | join(',') }}"
- zeppelin_server_url: "{{ groups.zeppelin[0] }}:9995"
- - metron-rest-env:
- metron_jdbc_driver: "org.h2.Driver"
- metron_jdbc_url: "jdbc:h2:file:~/metrondb"
- metron_jdbc_username: "root"
- metron_jdbc_password: "root"
- metron_jdbc_platform: "h2"
- - kibana-env:
- kibana_pid_dir: /var/run/kibana
- kibana_es_url: http://{{ groups.search[0] }}:9200
- kibana_log_dir: /var/log/kibana
- kibana_server_port: 5000
- kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
- kibana_server_host: 0.0.0.0
-
-blueprint:
- stack_name: HDP
- stack_version: "{{ hdp_stack }}"
- required_configurations: "{{ required_configurations }}"
- groups:
- - name : host_group_1
- cardinality: 1
- configurations: []
- components: "{{ metron_components }}"
- hosts: "{{ hdp_host_group }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/roles/ambari_config/vars/small_cluster.yml
deleted file mode 100644
index 4ec8458..0000000
--- a/metron-deployment/roles/ambari_config/vars/small_cluster.yml
+++ /dev/null
@@ -1,149 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
-app_timeline_server: [APP_TIMELINE_SERVER]
-hadoop_slave: [DATANODE, NODEMANAGER]
-spark_master: [SPARK_JOBHISTORYSERVER]
-storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
-storm_slave: [SUPERVISOR]
-kafka_broker: [KAFKA_BROKER]
-zookeeper_master: [ZOOKEEPER_SERVER]
-hbase_master: [HBASE_MASTER]
-hbase_slave: [HBASE_REGIONSERVER]
-hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
-es_master: [ES_MASTER]
-es_slave: [ES_SLAVE]
-kibana_master: [KIBANA_MASTER]
-metron_indexing: [METRON_INDEXING]
-metron_profiler: [METRON_PROFILER]
-metron_enrichment_master : [METRON_ENRICHMENT_MASTER]
-metron_parsers : [METRON_PARSERS]
-metron_rest: [METRON_REST]
-metron_management_ui: [METRON_MANAGEMENT_UI]
-metron_alerts_ui: [METRON_ALERTS_UI]
-
-master_1_components: "{{ hadoop_master | union(hadoop_clients) | union(es_slave) }}"
-master_1_host:
- - "{{groups.ambari_slave[0]}}"
-master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) | union(app_timeline_server) | union(es_slave) }}"
-master_2_host:
- - "{{groups.ambari_slave[1]}}"
-metron_components: >
- {{ metron_indexing | union(metron_profiler) | union(metron_enrichment_master) | union(metron_parsers) | union(metron_rest) | union(metron_management_ui) | union(metron_alerts_ui) | union(hadoop_slave) | union(storm_slave) |
- union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}
-metron_host:
- - "{{ groups.metron[0] }}"
-web_components: "{{ kibana_master | union(es_master) }}"
-web_host:
- - "{{ groups.web[0] }}"
-slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"
-
-cluster_name: "metron"
-blueprint_name: "metron_blueprint"
-
-configurations:
- - zoo.cfg:
- dataDir: '{{ zookeeper_data_dir | default("/hadoop/zookeeper") }}'
- - hadoop-env:
- namenode_heapsize: 2048
- dtnode_heapsize: 1024
- - hbase-env:
- hbase_regionserver_heapsize: 1024
- hbase_master_heapsize: 1024
- - hdfs-site:
- dfs.namenode.checkpoint.dir: '{{ namenode_checkpoint_dir | default("/hadoop/hdfs/namesecondary") }}'
- dfs.namenode.name.dir: '{{ namenode_name_dir | default("/hadoop/hdfs/namenode") }}'
- dfs.datanode.data.dir: '{{ datanode_data_dir | default("/hadoop/hdfs/data" ) }}'
- dfs.journalnode.edits.dir: '{{ journalnode_edits_dir | default("/hadoop/hdfs/journalnode") }}'
- - mapred-site:
- mapreduce.jobhistory.recovery.store.leveldb.path : '{{ jhs_recovery_store_ldb_path | default("/hadoop/mapreduce/jhs") }}'
- mapreduce.map.memory.mb : '{{ mapred_map_mem_mb }}'
- mapreduce.reduce.memory.mb : '{{ mapred_reduce_mem_mb }}'
- - yarn-site:
- yarn.nodemanager.local-dirs : '{{ nodemanager_local_dirs| default("/hadoop/yarn/local") }}'
- yarn.timeline-service.leveldb-timeline-store.path: '{{ timeline_ldb_store_path | default("/hadoop/yarn/timeline") }}'
- yarn.timeline-service.leveldb-state-store.path: '{{ timeline_ldb_state_path| default("/hadoop/yarn/timeline") }}'
- yarn.nodemanager.log-dirs: '{{ nodemanager_log_dirs| default("/hadoop/yarn/log") }}'
- yarn.nodemanager.resource.memory-mb : '{{ nodemanager_mem_mb }}'
- - storm-site:
- supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
- storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
- topology.classpath: '{{ topology_classpath }}'
- - kafka-broker:
- log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
- - metron-rest-env:
- metron_spring_profiles_active: "dev"
- - metron-env:
- parsers: "bro,snort,yaf"
- - elastic-site:
- index_number_of_shards: 2
- index_number_of_replicas: 1
- zen_discovery_ping_unicast_hosts: "[ {{ groups.web[0] }}, {{ groups.search | join(', ') }} ]"
- gateway_recover_after_data_nodes: 1
- network_host: "[ _local_, {{ elasticsearch_network_interface }} ]"
-
-required_configurations:
- - metron-env:
- storm_rest_addr: "http://{{ groups.ambari_slave[1] }}:8744"
- es_hosts: "{{ groups.web[0] }},{{ groups.search | join(',') }}"
- zeppelin_server_url: "{{ groups.zeppelin[0] }}"
- - metron-rest-env:
- metron_jdbc_driver: "org.h2.Driver"
- metron_jdbc_url: "jdbc:h2:file:~/metrondb"
- metron_jdbc_username: "root"
- metron_jdbc_password: "root"
- metron_jdbc_platform: "h2"
- - kibana-env:
- kibana_pid_dir: /var/run/kibana
- kibana_es_url: http://{{ groups.web[0] }}:9200
- kibana_log_dir: /var/log/kibana
- kibana_server_port: 5000
- kibana_default_application: "dashboard/AV-YpDmwdXwc6Ua9Muh9"
- kibana_server_host: 0.0.0.0
-
-blueprint:
- stack_name: HDP
- stack_version: "{{ hdp_stack }}"
- required_configurations: "{{ required_configurations }}"
- groups:
- - name : master_1
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ master_1_components }}"
- hosts: "{{ master_1_host }}"
- - name : master_2
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ master_2_components }}"
- hosts: "{{ master_2_host }}"
- - name : metron
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ metron_components }}"
- hosts: "{{ metron_host }}"
- - name : web
- cardinality: 1
- configuration: [] # configuration not yet implemented
- components: "{{ web_components }}"
- hosts: "{{ web_host }}"
- - name: slaves
- cardinality: 1+
- configuration: [] # configuration not yet implemented
- components: "{{ slave_components }}"
- hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) | difference(groups.metron[0]) | difference(groups.web[0]) }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_gather_facts/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_gather_facts/meta/main.yml b/metron-deployment/roles/ambari_gather_facts/meta/main.yml
deleted file mode 100644
index 61197e3..0000000
--- a/metron-deployment/roles/ambari_gather_facts/meta/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - epel
- - python-pip
- - httplib2
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/roles/ambari_gather_facts/tasks/main.yml
deleted file mode 100644
index 2b37eec..0000000
--- a/metron-deployment/roles/ambari_gather_facts/tasks/main.yml
+++ /dev/null
@@ -1,234 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# cluster_name
-#
-- name: "Ask Ambari: cluster_name"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: cluster_name_response
- when: cluster_name is undefined
-
-- set_fact:
- cluster_name: "{{ (cluster_name_response.content | from_json)['items'][0].Clusters.cluster_name }}"
- when: cluster_name is undefined
-
-#
-# namenode_host
-#
-- name: "Ask Ambari: namenode_host"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: namenode_hosts_response
- when: namenode_host is undefined
-
-- set_fact:
- namenode_host: "{{ (namenode_hosts_response.content | from_json).host_components[0].HostRoles.host_name }}"
- when: namenode_host is undefined
-
-#
-# core_site_tag
-#
-- name: "Ask Ambari: core_site_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: core_site_tag_response
- when: core_site_tag is undefined
-
-- set_fact:
- core_site_tag: "{{ (core_site_tag_response.content | from_json).HostRoles.actual_configs['core-site'].default }}"
- when: core_site_tag is undefined
-
-#
-# hdfs_url
-#
-- name: "Ask Ambari: hdfs_url"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: core_site_response
- when: hdfs_url is undefined
-
-- set_fact:
- hdfs_url: "{{ (core_site_response.content | from_json)['items'][0].properties['fs.defaultFS'] }}"
- when: hdfs_url is undefined
-
-#
-# kafka_broker_hosts
-#
-- name: "Ask Ambari: kafka_broker_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: kafka_broker_hosts_response
- when: kafka_broker_hosts is undefined
-
-- set_fact:
- kafka_broker_hosts: "{{ (kafka_broker_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: kafka_broker_hosts is undefined
-
-#
-# kafka_broker_tag
-#
-- name: "Ask Ambari: kafka_broker_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: kafka_broker_tag_response
- when: kafka_broker_tag is undefined
-
-- set_fact:
- kafka_broker_tag: "{{ (kafka_broker_tag_response.content | from_json).HostRoles.actual_configs['kafka-broker'].default }}"
- when: kafka_broker_tag is undefined
-
-#
-# kafka_broker_port
-#
-- name: "Ask Ambari: kafka_broker_port"
- shell: >
- curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H "X-Requested-By: ambari" "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}" | python -c 'import sys, json; print json.load(sys.stdin)["items"][0]["properties"]["listeners"]'
- args:
- warn: false
- register: kafka_broker_port_response
- when: (kafka_broker_url is undefined) or (kafka_broker_port is undefined)
-
-- set_fact:
- kafka_broker_port: "{{ kafka_broker_port_response.stdout_lines[0] | replace('PLAINTEXT://localhost:', '')}}"
- when: kafka_broker_port is undefined
-
-- set_fact:
- kafka_broker_url: "{% for host in kafka_broker_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ kafka_broker_port }}{% endfor %}"
- when: kafka_broker_url is undefined
-
-#
-# zookeeper_hosts
-#
-- name: "Ask Ambari: zookeeper_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: zookeeper_hosts_response
- when: zookeeper_hosts is undefined
-
-- set_fact:
- zookeeper_hosts: "{{ (zookeeper_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: zookeeper_hosts is undefined
-
-#
-# zookeeper_tag
-#
-- name: "Ask Ambari: zookeeper_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: zookeeper_tag_response
- when: zookeeper_tag is undefined
-
-- set_fact:
- zookeeper_tag: "{{ (zookeeper_tag_response.content | from_json).HostRoles.actual_configs['zoo.cfg'].default }}"
- when: zookeeper_tag is undefined
-
-#
-# zookeeper_url, zookeeper_port
-#
-- name: "Ask Ambari: zookeeper_url, zookeeper_port"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: zookeeper_properties_response
- when: zookeeper_url is undefined or zookeeper_port is undefined
-
-- set_fact:
- zookeeper_port: "{{ (zookeeper_properties_response.content | from_json)['items'][0].properties['clientPort'] }}"
- when: zookeeper_port is undefined
-
-- set_fact:
- zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}"
- when: zookeeper_url is undefined
-
-- name: "Ask Ambari: metron_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: metron_hosts_response
- when: metron_hosts is undefined
-
-- set_fact:
- metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: metron_hosts is undefined
-
-- name: "Ask Ambari: kibana_hosts"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: kibana_hosts_response
- when: kibana_hosts is undefined
-
-- set_fact:
- kibana_hosts: "{{ (kibana_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
- when: kibana_hosts is undefined
-
-
-#
-# debug output
-#
-- name: debug
- debug:
- msg: "zookeeper_port = {{ zookeeper_port }},
- zookeeper_hosts = {{ zookeeper_hosts }},
- zookeeper_url = {{ zookeeper_url }},
- kafka_broker_port = {{ kafka_broker_port }},
- kafka_broker_hosts = {{ kafka_broker_hosts }},
- kafka_broker_url = {{ kafka_broker_url }},
- metron_hosts = {{ metron_hosts }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_master/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_master/defaults/main.yml b/metron-deployment/roles/ambari_master/defaults/main.yml
deleted file mode 100644
index bc6c08c..0000000
--- a/metron-deployment/roles/ambari_master/defaults/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-ambari_server_mem: 2048
-ambari_mpack_version: 0.4.3.0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_master/tasks/main.yml b/metron-deployment/roles/ambari_master/tasks/main.yml
deleted file mode 100644
index bc8e837..0000000
--- a/metron-deployment/roles/ambari_master/tasks/main.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-# tasks file for ambari_master
-- name: Install ambari server
- yum:
- name: ambari-server
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Set Ambari Server Max Memory
- replace:
- dest: /var/lib/ambari-server/ambari-env.sh
- regexp: "\ -Xmx2048m\ "
- replace: " -Xmx{{ ambari_server_mem }}m "
- backup: no
-
-- name: Setup ambari server
- shell: ambari-server setup -s && touch /etc/ambari-server/configured creates=/etc/ambari-server/configured
- register: ambari_server_setup
- failed_when: "(ambari_server_setup.stderr is defined and ambari_server_setup.stderr != '') or 'FATAL' in ambari_server_setup.stdout"
-
-- name: Copy MPack to Ambari Host
- copy:
- src: "{{ playbook_dir }}/../packaging/ambari/metron-mpack/target/metron_mpack-{{ ambari_mpack_version }}.tar.gz"
- dest: /tmp
-
-- name: Install MPack on Ambari Host
- shell: ambari-server install-mpack --mpack=/tmp/metron_mpack-0.4.3.0.tar.gz
- args:
- creates: /var/lib/ambari-server/resources/mpacks/metron-ambari.mpack-{{ ambari_mpack_version }}/addon-services
-
-- name: start ambari server
- service:
- name: ambari-server
- state: restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/defaults/main.yml b/metron-deployment/roles/ambari_slave/defaults/main.yml
deleted file mode 100644
index f3cb4c5..0000000
--- a/metron-deployment/roles/ambari_slave/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-ambari_installation_user: "root"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/files/hostname.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/files/hostname.sh b/metron-deployment/roles/ambari_slave/files/hostname.sh
deleted file mode 100644
index cc8c1cd..0000000
--- a/metron-deployment/roles/ambari_slave/files/hostname.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/bin/sh
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-echo {{ inventory_hostname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/meta/main.yml b/metron-deployment/roles/ambari_slave/meta/main.yml
deleted file mode 100644
index ddf6aa9..0000000
--- a/metron-deployment/roles/ambari_slave/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ambari_slave/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ambari_slave/tasks/main.yml b/metron-deployment/roles/ambari_slave/tasks/main.yml
deleted file mode 100644
index ca81a7a..0000000
--- a/metron-deployment/roles/ambari_slave/tasks/main.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: ../roles/metron-rpms/tasks/create_directory.yml
-- include: ../roles/metron-rpms/tasks/create_repo.yml
-
-- name: Install ambari-agent
- yum:
- name: ambari-agent
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Create ambari-agent hostname script
- template:
- src: "../roles/ambari_slave/files/hostname.sh"
- dest: "/var/lib/ambari-agent/hostname.sh"
- mode: 0744
- owner: "{{ ambari_installation_user }}"
- group: "{{ ambari_installation_user }}"
-
-- name: Configure ambari-server hostname in ambari-agent configuration
- lineinfile:
- dest: /etc/ambari-agent/conf/ambari-agent.ini
- regexp: "{{ item.regexp }}"
- line: "{{ item.line }}"
- insertafter: "{{ item.insertafter }}"
- backup: yes
- with_items:
- - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
- - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
-
-- name: Ensure ambari-agent is running
- service:
- name: ambari-agent
- state: restarted
- enabled: yes
-
-- name : Wait for agent to register
- command : sleep 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/meta/main.yml b/metron-deployment/roles/bro/meta/main.yml
deleted file mode 100644
index df226e7..0000000
--- a/metron-deployment/roles/bro/meta/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - libselinux-python
- - build-tools
- - kafka-client
- - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/bro.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/bro.yml b/metron-deployment/roles/bro/tasks/bro.yml
deleted file mode 100644
index 222ef0e..0000000
--- a/metron-deployment/roles/bro/tasks/bro.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download bro
- get_url:
- url: "https://www.bro.org/downloads/bro-{{ bro_version }}.tar.gz"
- dest: "/tmp/bro-{{ bro_version }}.tar.gz"
-
-- name: Extract bro tarball
- unarchive:
- src: "/tmp/bro-{{ bro_version }}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/bro-{{ bro_version }}"
-
-- name: Compile and Install bro
- shell: "{{ item }}"
- environment:
- CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
- CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
- args:
- chdir: "/tmp/bro-{{ bro_version }}"
- creates: "{{ bro_home }}/bin/bro"
- with_items:
- - "./configure --prefix={{ bro_home }}"
- - make
- - make install
-
-- name: Configure bro
- lineinfile:
- dest: "{{ bro_home }}/etc/node.cfg"
- regexp: '^interface=.*$'
- line: 'interface={{ sniff_interface }}'
-
-- name: Install bro configuration
- shell: "{{ bro_home }}/bin/broctl install"
-
-- name: Bro Cronjob
- cron:
- name: Bro Cron
- minute: "{{ bro_crontab_minutes }}"
- job: "{{ item }}"
- with_items:
- - "{{ bro_crontab_job }}"
- - "{{ bro_clean_job }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/dependencies.yml b/metron-deployment/roles/bro/tasks/dependencies.yml
deleted file mode 100644
index 3cd3bae..0000000
--- a/metron-deployment/roles/bro/tasks/dependencies.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install yum repositories
- yum: name={{ item }} update_cache=yes
- with_items:
- - centos-release-scl
-
-- name: Install prerequisites
- yum: name={{ item }}
- with_items:
- - cmake
- - make
- - gcc
- - gcc-c++
- - flex
- - bison
- - libpcap
- - libpcap-devel
- - openssl-devel
- - python-devel
- - swig
- - zlib-devel
- - perl
- - crontabs
- - net-tools
- - devtoolset-4-gcc
- - devtoolset-4-gcc-c++
- - python27
- - rh-git29
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/librdkafka.yml b/metron-deployment/roles/bro/tasks/librdkafka.yml
deleted file mode 100644
index 652d319..0000000
--- a/metron-deployment/roles/bro/tasks/librdkafka.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download librdkafka
- get_url:
- url: "{{ librdkafka_url }}"
- dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
-
-- name: Extract librdkafka tarball
- unarchive:
- src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/librdkafka-{{ librdkafka_version }}"
-
-- name: Compile and install librdkafka
- shell: "{{ item }}"
- args:
- chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
- creates: "{{ librdkafka_home }}/lib/librdkafka.so"
- with_items:
- - rm -rf build/
- - "./configure --prefix={{ librdkafka_home }}"
- - make
- - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/main.yml b/metron-deployment/roles/bro/tasks/main.yml
deleted file mode 100644
index 440e4b5..0000000
--- a/metron-deployment/roles/bro/tasks/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: librdkafka.yml
-- include: bro.yml
-- include: metron-bro-plugin-kafka.yml
-- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml b/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
deleted file mode 100644
index 7043387..0000000
--- a/metron-deployment/roles/bro/tasks/metron-bro-plugin-kafka.yml
+++ /dev/null
@@ -1,49 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install bro-pkg
- environment:
- LD_LIBRARY_PATH: "{{ python27_lib }}"
- command: "{{ python27_bin }}/pip install bro-pkg"
-
-- name: Configure bro-pkg
- environment:
- PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
- LD_LIBRARY_PATH: "{{ python27_lib }}"
- command: "{{ python27_bin }}/bro-pkg autoconfig"
-
-- name: Install the metron-bro-plugin-kafka package
- environment:
- PATH: "{{ git29_bin }}:{{ bro_bin }}:{{ ansible_env.PATH }}"
- LD_LIBRARY_PATH: "{{ httpd24_lib }}:{{ python27_lib }}"
- CXX: /opt/rh/devtoolset-4/root/usr/bin/g++
- CC: /opt/rh/devtoolset-4/root/usr/bin/gcc
- command: "{{ python27_bin }}/bro-pkg install apache/metron-bro-plugin-kafka --force --version {{ metron_bro_plugin_kafka_version }}"
-
-- name: Configure bro-kafka plugin
- lineinfile:
- dest: "{{ bro_home }}/share/bro/site/local.bro"
- line: "{{ item }}"
- with_items:
- - "@load packages"
- - "redef Kafka::logs_to_send = set(HTTP::LOG, DNS::LOG);"
- - "redef Kafka::topic_name = \"{{ bro_topic }}\";"
- - "redef Kafka::tag_json = T;"
- - "redef Kafka::kafka_conf = table([\"metadata.broker.list\"] = \"{{ kafka_broker_url }}\");"
-
-- name: Deploy bro configuration changes
- shell: "{{ bro_bin }}/broctl deploy"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/tasks/nic.yml b/metron-deployment/roles/bro/tasks/nic.yml
deleted file mode 100644
index 6053618..0000000
--- a/metron-deployment/roles/bro/tasks/nic.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
[28/50] [abbrv] metron git commit: METRON-1410 Some more upgrade
fallout... Can'
t restart Metron Indexing. (ottobackwards) closes apache/metron#901
Posted by rm...@apache.org.
METRON-1410 Some more upgrade fallout... Can't restart Metron Indexing. (ottobackwards) closes apache/metron#901
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/0c6aad8c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/0c6aad8c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/0c6aad8c
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 0c6aad8c7c7affd6c898e1dd7663833a71ba63de
Parents: 6f26799
Author: ottobackwards <ot...@gmail.com>
Authored: Sat Jan 27 17:22:46 2018 -0500
Committer: otto <ot...@apache.org>
Committed: Sat Jan 27 17:22:46 2018 -0500
----------------------------------------------------------------------
.../package/scripts/enrichment_commands.py | 36 +++++---------------
.../package/scripts/indexing_commands.py | 18 ++--------
.../CURRENT/package/scripts/metron_service.py | 26 ++++++++++++++
.../package/scripts/profiler_commands.py | 26 ++++----------
4 files changed, 45 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
index 90a690e..f9ec547 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/enrichment_commands.py
@@ -17,14 +17,15 @@ limitations under the License.
import os
import time
-from datetime import datetime
+
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
+from resource_management.core.resources.system import Execute
import metron_service
from metron_security import kinit
+
# Wrap major operations and functionality in this class
class EnrichmentCommands:
__params = None
@@ -184,31 +185,12 @@ class EnrichmentCommands:
def create_hbase_tables(self):
Logger.info("Creating HBase Tables")
- if self.__params.security_enabled:
- kinit(self.__params.kinit_path_local,
- self.__params.hbase_keytab_path,
- self.__params.hbase_principal_name,
- execute_user=self.__params.hbase_user)
-
- cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_enrichment_cmd = cmd.format(self.__params.enrichment_hbase_table, self.__params.enrichment_hbase_cf)
- Execute(add_enrichment_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
- add_threatintel_cmd = cmd.format(self.__params.threatintel_hbase_table, self.__params.threatintel_hbase_cf)
- Execute(add_threatintel_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
+ metron_service.create_hbase_table(self.__params,
+ self.__params.enrichment_hbase_table,
+ self.__params.enrichment_hbase_cf)
+ metron_service.create_hbase_table(self.__params,
+ self.__params.threatintel_hbase_table,
+ self.__params.threatintel_hbase_cf)
Logger.info("Done creating HBase Tables")
self.set_hbase_configured()
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
index 33f45d4..c057b72 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/indexing_commands.py
@@ -113,21 +113,9 @@ class IndexingCommands:
def create_hbase_tables(self):
Logger.info("Creating HBase Tables for indexing")
- if self.__params.security_enabled:
- metron_security.kinit(self.__params.kinit_path_local,
- self.__params.hbase_keytab_path,
- self.__params.hbase_principal_name,
- execute_user=self.__params.hbase_user)
- cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_update_cmd = cmd.format(self.__params.update_hbase_table, self.__params.update_hbase_cf)
- Execute(add_update_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
+ metron_service.create_hbase_table(self.__params,
+ self.__params.update_hbase_table,
+ self.__params.update_hbase_cf)
Logger.info("Done creating HBase Tables for indexing")
self.set_hbase_configured()
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
index d8b8b10..330d3c0 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/metron_service.py
@@ -282,6 +282,32 @@ def check_kafka_topics(params, topics):
err_msg = "Missing Kafka topic; topic={0}".format(topic)
execute(cmd, user=params.kafka_user, err_msg=err_msg)
+
+def create_hbase_table(params, table, cf):
+ """
+ Creates an HBase table, if the table does not currently exist
+ :param params:
+ :param table: The name of the HBase table.
+ :param cf: The column family
+ :param user: The user to execute the command as
+ """
+ if params.security_enabled:
+ kinit(params.kinit_path_local,
+ params.hbase_keytab_path,
+ params.hbase_principal_name,
+ execute_user=params.hbase_user)
+ cmd = """if [[ $(echo \"exists '{0}'\" | hbase shell | grep 'not exist') ]]; \
+ then echo \"create '{0}','{1}'\" | hbase shell -n; fi"""
+ add_update_cmd = cmd.format(table, cf)
+ Execute(add_update_cmd,
+ tries=3,
+ try_sleep=5,
+ logoutput=False,
+ path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
+ user=params.hbase_user
+ )
+
+
def check_hbase_table(params, table):
"""
Validates that an HBase table exists. An exception is raised if the table
http://git-wip-us.apache.org/repos/asf/metron/blob/0c6aad8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
index 41cab06..b026a30 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/profiler_commands.py
@@ -18,13 +18,12 @@ limitations under the License.
import os
import time
-from datetime import datetime
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute, File
+from resource_management.core.resources.system import Execute
-import metron_service
import metron_security
+import metron_service
# Wrap major operations and functionality in this class
@@ -79,22 +78,11 @@ class ProfilerCommands:
metron_service.set_configured(self.__params.metron_user, self.__params.profiler_hbase_acl_configured_flag_file, "Setting HBase ACL configured to True for profiler")
def create_hbase_tables(self):
- Logger.info("Creating HBase table '{0}' for profiler".format(self.__params.profiler_hbase_table))
- if self.__params.security_enabled:
- metron_security.kinit(self.__params.kinit_path_local,
- self.__params.hbase_keytab_path,
- self.__params.hbase_principal_name,
- execute_user=self.__params.hbase_user)
- cmd = "echo \"create '{0}','{1}'\" | hbase shell -n"
- add_table_cmd = cmd.format(self.__params.profiler_hbase_table, self.__params.profiler_hbase_cf)
- Execute(add_table_cmd,
- tries=3,
- try_sleep=5,
- logoutput=False,
- path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin',
- user=self.__params.hbase_user
- )
-
+ Logger.info("Creating HBase table '{0}' for profiler".format(
+ self.__params.profiler_hbase_table))
+ metron_service.create_hbase_table(self.__params,
+ self.__params.profiler_hbase_table,
+ self.__params.profiler_hbase_cf)
self.set_hbase_configured()
Logger.info("Done creating HBase Tables for profiler")
[36/50] [abbrv] metron git commit: METRON-1435 Management UI cannot
save json objects in advanced config (merrimanr) closes apache/metron#917
Posted by rm...@apache.org.
METRON-1435 Management UI cannot save json objects in advanced config (merrimanr) closes apache/metron#917
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3d3c43c7
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3d3c43c7
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3d3c43c7
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 3d3c43c716b0a6d58b57d6d5ba83109bc49661e4
Parents: 567d106
Author: merrimanr <me...@gmail.com>
Authored: Fri Feb 2 08:37:29 2018 -0600
Committer: merrimanr <me...@apache.org>
Committed: Fri Feb 2 08:37:29 2018 -0600
----------------------------------------------------------------------
.../advanced-config-form.component.html | 2 +-
.../advanced-config-form.component.spec.ts | 34 ++++++++++++++++++++
.../advanced-config-form.component.ts | 20 +++++++++++-
3 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/3d3c43c7/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
index 1a30ee7..945be5f 100644
--- a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
+++ b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html
@@ -19,7 +19,7 @@
</div>
</div>
<div class="row mx-0">
- <div class="col-md-10 advanced-input"><input type="text" class="form-control" formControlName="{{key}}" [(ngModel)]="config[key]"></div>
+ <div class="col-md-10 advanced-input"><input type="text" class="form-control" formControlName="{{key}}" [ngModel]="displayValue(key)" (ngModelChange)="saveValue(key, $event)"></div>
<div class="col-md-2" (click)="removeConfig(key)">
<i class="fa fa-minus fa-4 icon-button" aria-hidden="true" ></i>
</div>
http://git-wip-us.apache.org/repos/asf/metron/blob/3d3c43c7/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
index a8f0ed0..ed80c54 100644
--- a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
+++ b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.spec.ts
@@ -136,6 +136,18 @@ describe('Component: AdvancedConfigFormComponent', () => {
expect(component.configForm.controls['field2'].value).toEqual('value2');
expect(component.configForm.controls['field3'].value).toEqual('value3');
+ component.newConfigKey = 'field1';
+ component.newConfigValue = '["newValue1"]';
+ component.saveNewConfig();
+ expect(Object.keys(component.config).length).toEqual(3);
+ expect(component.config['field1']).toEqual(['newValue1']);
+
+ component.newConfigKey = 'field1';
+ component.newConfigValue = '{"key":"newValue1"}';
+ component.saveNewConfig();
+ expect(Object.keys(component.config).length).toEqual(3);
+ expect(component.config['field1']).toEqual({key: 'newValue1'});
+
component.removeConfig('field1');
expect(Object.keys(component.config).length).toEqual(2);
expect(component.config['field2']).toEqual('value2');
@@ -146,6 +158,28 @@ describe('Component: AdvancedConfigFormComponent', () => {
expect(component.configForm.controls['newConfigValue'].value).toEqual('enter value');
expect(component.configForm.controls['field2'].value).toEqual('value2');
expect(component.configForm.controls['field3'].value).toEqual('value3');
+
+
}));
+ it('verify display and save values', async(() => {
+ let component: AdvancedConfigFormComponent = fixture.componentInstance;
+ component.config = {'field1': 'value1', 'field2': 'value2'};
+ component.ngOnInit();
+
+ expect(component.displayValue('field1')).toEqual('value1');
+
+ component.saveValue('field1', '["value1","value2"]');
+ expect(component.config['field1']).toEqual(['value1', 'value2']);
+ expect(component.displayValue('field1')).toEqual('["value1","value2"]');
+
+ component.saveValue('field1', '["value1","value2"');
+ expect(component.config['field1']).toEqual('["value1","value2"');
+ expect(component.displayValue('field1')).toEqual('["value1","value2"');
+
+ component.saveValue('field1', '{"key1":"value1"}');
+ expect(component.config['field1']).toEqual({'key1': 'value1'});
+ expect(component.displayValue('field1')).toEqual('{"key1":"value1"}');
+ }));
+
});
http://git-wip-us.apache.org/repos/asf/metron/blob/3d3c43c7/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts
----------------------------------------------------------------------
diff --git a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts
index f363391..5da9d48 100644
--- a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts
+++ b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts
@@ -83,7 +83,7 @@ export class AdvancedConfigFormComponent implements OnInit, OnChanges {
}
if (this.newConfigKey !== 'enter field' && this.newConfigValue !== 'enter value') {
let keyExists = this.config[this.newConfigKey] !== undefined;
- this.config[this.newConfigKey] = this.newConfigValue;
+ this.saveValue(this.newConfigKey, this.newConfigValue);
if (keyExists) {
this.newConfigKey = 'enter field';
this.newConfigValue = 'enter value';
@@ -107,4 +107,22 @@ export class AdvancedConfigFormComponent implements OnInit, OnChanges {
this.configForm.removeControl(key);
}
+ displayValue(key: string): string {
+ let value = this.config[key];
+ if (Array.isArray(value) || value instanceof Object) {
+ return JSON.stringify(value);
+ } else {
+ return value;
+ }
+ }
+
+ saveValue(key: string, value: string) {
+ try {
+ this.config[key] = JSON.parse(value);
+ } catch (err) {
+ this.config[key] = value;
+ }
+
+ }
+
}
[32/50] [abbrv] metron git commit: METRON-1391 Typos in
Documentation/Examples within metron-management/README.md (havran via
ottobackwards) closes apache/metron#890
Posted by rm...@apache.org.
METRON-1391 Typos in Documentation/Examples within metron-management/README.md (havran via ottobackwards) closes apache/metron#890
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/08745719
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/08745719
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/08745719
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 08745719120fd8d42e36c728a405966ad64f20b7
Parents: 1c9437c
Author: havran <ha...@gmail.com>
Authored: Tue Jan 30 10:40:04 2018 -0500
Committer: otto <ot...@apache.org>
Committed: Tue Jan 30 10:40:04 2018 -0500
----------------------------------------------------------------------
metron-platform/metron-management/README.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/08745719/metron-platform/metron-management/README.md
----------------------------------------------------------------------
diff --git a/metron-platform/metron-management/README.md b/metron-platform/metron-management/README.md
index c4b9555..812583c 100644
--- a/metron-platform/metron-management/README.md
+++ b/metron-platform/metron-management/README.md
@@ -898,7 +898,7 @@ Returns: A Map associated with the indicator and enrichment type. Empty otherwi
[Stellar]>>> non_us := whois_info.home_country != 'US'
[Stellar]>>> is_local := IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')
[Stellar]>>> is_both := whois_info.home_country != 'US' && IN_SUBNET( if IS_IP(ip_src_addr) then ip_src_addr else NULL, '192.168.0.0/21')
-[Stellar]>>> rules := [ { 'name' : 'is non-us', 'rule' : SHELL_GET_EXPRESSION('non_us'), 'score' : 10 } , { 'name' : 'is local', 'rule' : SHELL_GET_EXPRESSION('is_local '), 'score' : 20 } , { 'name' : 'both non-us and local', 'comment' : 'union of both rules.', 'rule' : SHELL_GET_EXPRESSION('is_both'), 'score' : 50 } ]
+[Stellar]>>> rules := [ { 'name' : 'is non-us', 'rule' : SHELL_GET_EXPRESSION('non_us'), 'score' : 10 } , { 'name' : 'is local', 'rule' : SHELL_GET_EXPRESSION('is_local'), 'score' : 20 } , { 'name' : 'both non-us and local', 'comment' : 'union of both rules.', 'rule' : SHELL_GET_EXPRESSION('is_both'), 'score' : 50 } ]
[Stellar]>>> # Now that we have our rules staged, we can add them to our config.
[Stellar]>>> squid_enrichment_config_new := THREAT_TRIAGE_ADD( squid_enrichment_config_new, rules )
[Stellar]>>> THREAT_TRIAGE_PRINT(squid_enrichment_config_new)
@@ -1020,7 +1020,7 @@ SION('is_both') ] )
1. Add a few triage rules.
```
- [Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10",
+ [Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule1", "rule":"value>10", "score":10})
```
```
[Stellar]>>> THREAT_TRIAGE_ADD(t, {"name":"rule2", "rule":"value>20", "score":20})
[23/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/files/bro.out
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/files/bro.out b/metron-deployment/ansible/roles/sensor-stubs/files/bro.out
new file mode 100644
index 0000000..09cacfd
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/files/bro.out
@@ -0,0 +1,1346 @@
+{"dns": {"ts":1484167797.685113,"uid":"C6MvbX3LOBTaZxwFCd","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484167798.055447,"uid":"CgA8NA4MlyRJmknKpf","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F7E6kW3Y9gaoHrgMCf"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167798.772618,"uid":"Cf322m3qSDKAsR8hTi","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Ftwsgz49riOg3FHK8l"],"resp_mime_types":["application/x-shockwave-flash"]}}
+{"dns": {"ts":1484167798.493574,"uid":"CZhu2h4SgVZWdW1aQl","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"dns": {"ts":1484167798.494185,"uid":"Czlibt2NgLkfLp7FPh","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484167799.934294,"uid":"CsWciw2WfERVM0Aczg","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fvq86i4oLSojHBFLSj"]}}
+{"http": {"ts":1484167800.359204,"uid":"C4airn3x7y3KVhZy8i","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJMtKEAMFuMvzDFJ7"],"resp_mime_types":["application/x-dosexec"]}}
+{"http": {"ts":1484167802.758608,"uid":"CJFa5h4HKSqVViZwbb","id.orig_h":"192.168.138.158","id.orig_p":49190,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b2566564b3ba1a38e61c83957a7dbcd5","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167803.072425,"uid":"CMOdJA4OzlvgzCK2qf","id.orig_h":"192.168.138.158","id.orig_p":49191,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?3a08b0be8322c244f5a1cb9c1057d941","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167803.402485,"uid":"CJVA893e60mcz43Jrj","id.orig_h":"192.168.138.158","id.orig_p":49192,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?d71e0bd86db9587158745a986a4b3606","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167804.009067,"uid":"C1rmUO2zDsIbgBR8Ik","id.orig_h":"192.168.138.158","id.orig_p":49193,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?34eaf8bd50d85d8c6baacb45f0a7b22e","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167804.312623,"uid":"CCTaln3ggV4dOqGETi","id.orig_h":"192.168.138.158","id.orig_p":49194,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?60dbe33b908e0086292196ef001816bc","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167804.557307,"uid":"CK6DaM2pfjgwi3pY8j","id.orig_h":"192.168.138.158","id.orig_p":49197,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?t=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["Fkypgg28hizg2EwJRk"],"orig_mime_types":["text/plain"],"resp_fuids":["FqQOjx3rFxTb4RSHE9"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167804.624712,"uid":"Clghht2drNjg3G5dPh","id.orig_h":"192.168.138.158","id.orig_p":49196,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?51424ddd486ff06861fceed24e86b329","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"dns": {"ts":1484167804.053752,"uid":"CoiTkw2sb9stNr10zg","id.orig_h":"192.168.138.158","id.orig_p":53571,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":15553,"query":"ip-addr.es","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["188.165.164.184"],"TTLs":[21599.0],"rejected":false}}
+{"dns": {"ts":1484167804.472938,"uid":"CJodZl3aVCrbHCw7xk","id.orig_h":"192.168.138.158","id.orig_p":61720,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":23625,"query":"runlove.us","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["204.152.254.221"],"TTLs":[14069.0],"rejected":false}}
+{"dns": {"ts":1484167804.737519,"uid":"CkV7Z23iTRHYS1MxCh","id.orig_h":"192.168.138.158","id.orig_p":50509,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":6088,"query":"kritischerkonsum.uni-koeln.de","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484167804.961078,"uid":"CBwIqk3VHRfD9CapGl","id.orig_h":"192.168.138.158","id.orig_p":56753,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":41589,"query":"comarksecurity.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["72.34.49.86"],"TTLs":[13888.0],"rejected":false}}
+{"http": {"ts":1484167805.046633,"uid":"CLfH0q2XJiBH0gUngj","id.orig_h":"192.168.138.158","id.orig_p":49198,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?c=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["F5sGcb3aKKVdqlR4l8"],"orig_mime_types":["text/plain"],"resp_fuids":["FfjQv01KPJAoEcMH1b"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167807.20478,"uid":"CgI9Lp32cTchxqp8Wk","id.orig_h":"192.168.138.158","id.orig_p":49199,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?l=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FDpZNy3tiCh1cjvs19"],"orig_mime_types":["text/plain"],"resp_fuids":["FCCDfF1umBiOBkbAl3"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167807.449353,"uid":"CuowPb45wYWpb50JTe","id.orig_h":"192.168.138.158","id.orig_p":49200,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?t=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":996,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FPM0BMoMv2KB8kbg6"],"orig_mime_types":["text/plain"],"resp_fuids":["F8YBeS1viVGE8sJCOe"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167810.18734,"uid":"ClAWUw4JlQ6WlHFWCc","id.orig_h":"192.168.138.158","id.orig_p":49201,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FMQqcF3hxESuoUxoAi"],"orig_mime_types":["text/plain"],"resp_fuids":["FGxTQj2tNlpA4iyAM7"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167810.407854,"uid":"CTC5QZ37i6qEO6MHef","id.orig_h":"192.168.138.158","id.orig_p":49202,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":45662,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FbWILCRYdYGyU9ZRk"],"orig_mime_types":["text/plain"],"resp_fuids":["FS225I2iUF28vOdi7i"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167816.579111,"uid":"CzJzUp2mgm3oMfTpxk","id.orig_h":"192.168.138.158","id.orig_p":49203,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?f=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["F7wKbu2LQZ2q8jSfc5"],"orig_mime_types":["text/plain"],"resp_fuids":["FF9gh04PL6B9Cvlbbl"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167816.846125,"uid":"CozvfM21cOxjQJvB8j","id.orig_h":"192.168.138.158","id.orig_p":49204,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FjWkDS5TCYn9GYRCd"],"orig_mime_types":["text/plain"],"resp_fuids":["FgMrsN3zfIRJ9ipWq6"],"resp_mime_types":["text/plain"]}}
+{"dns": {"ts":1484167810.995898,"uid":"CC18jH3AC7y8NPST2b","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"http": {"ts":1484167804.326254,"uid":"CwFhR03V5JXOYF8Ppk","id.orig_h":"192.168.138.158","id.orig_p":49195,"id.resp_h":"188.165.164.184","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ip-addr.es","uri":"/","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"tags":[]}}
+{"dns": {"ts":1484167842.890717,"uid":"CTvVE015EfMtdMxkU7","id.orig_h":"192.168.138.158","id.orig_p":50329,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":5810,"query":"7oqnsnzwwnm6zb7y.gigapaysun.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["95.163.121.204"],"TTLs":[14277.0],"rejected":false}}
+{"http": {"ts":1484167843.102031,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":3289,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fmv6Ap2EAcThJKped6"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167845.759905,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":4492,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FPQd6A1L85pPPCM7ya"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167847.5693,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/it.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":552,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fitq9q43ZupDyYoAyk"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167847.568708,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/us.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":825,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FSuqni2XAHJRnN8bYh"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167847.760685,"uid":"Cg6qof2daZW7072Gq4","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":240,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FTT0j8hWM1ENBWINg"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167847.759652,"uid":"CemTNB1OFxbrBn2wD2","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/picture.php?k=11iqmfg\u0026b7f2a994c3eaaf014608b272c46cf764","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":1823,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FKu4gy3zUcygg980ee"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167847.760176,"uid":"Cdg2Cf1BnvStDcNm44","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/es.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":634,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FoY2NUzH7asqxomge"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167847.776105,"uid":"CUrJ3S149MGwkEQcc","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/de.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":534,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fv44d82Oy08X20w3jh"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167849.341698,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/fr.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":694,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F7znd92YPrDeWJkLWb"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167849.558631,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":242,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FKJ5Y92fddrzBWKCb8"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167849.59428,"uid":"Cg6qof2daZW7072Gq4","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":239,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F3R4B2XMTLO3hSure"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167849.692474,"uid":"CemTNB1OFxbrBn2wD2","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":237,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FOoUNQ3OUe0r3e9Ewa"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167851.825173,"uid":"Cdg2Cf1BnvStDcNm44","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/favicon.ico","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":318,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F0ASzM1opxGAKE6oMe"],"resp_mime_types":["image/x-icon"]}}
+{"http": {"ts":1484167856.767294,"uid":"CUrJ3S149MGwkEQcc","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"POST","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":14,"response_body_len":14641,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FcUgiOBXtoqaQLmed"],"orig_mime_types":["text/plain"],"resp_fuids":["Fp1E561lGpI5pr3S8e"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167859.506315,"uid":"C55LNNhjZ7ttzams8","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":304,"status_msg":"Not Modified","tags":[]}}
+{"http": {"ts":1484167861.613787,"uid":"C0XtwFSGVX0paqsq9","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/bitcoin.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":5523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FZv62C25nyrRv26Mhl"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167861.614577,"uid":"Cg6qof2daZW7072Gq4","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/button_pay.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":727,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FICI2E4Vfpq8wOCTGc"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167798.787714,"uid":"CK8vN03Rc0FuI0R6qk","id.orig_h":"192.168.138.158","id.orig_p":49186,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":121635,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F4hDsd3d3xB7Rhef5i"],"resp_mime_types":["text/html"]}}
+{"dns": {"ts":1484167890.789624,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484167891.791607,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484167894.797258,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484167903.814905,"uid":"C3xvTs1NUXOIsMaj55","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"http": {"ts":1484167923.810197,"uid":"CaSyqTwPFwYZLa4ra","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FmsYkt2F76v7XUBNAd"],"resp_mime_types":["text/html"]}}
+{"dns": {"ts":1484167923.441012,"uid":"C21R6O7H0Kzv5Fdd6","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484167924.511788,"uid":"CRGLdEasAJUDL8Tu4","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FHMpUl2B1lUkpzZoQi"],"resp_mime_types":["application/x-shockwave-flash"]}}
+{"dns": {"ts":1484167924.233822,"uid":"CwNs81Nmjja4ubh79","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"dns": {"ts":1484167924.234067,"uid":"C5GlhlwTDTSQ0b7T7","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484167925.62215,"uid":"Cprvbi2qWL0ZpQ3hT1","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FiZAai2ofPvEPp5aC8"]}}
+{"http": {"ts":1484167926.026722,"uid":"CWFPu12Z6Poy9LL6q3","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FMtkgchK5p70TpU2"],"resp_mime_types":["application/x-dosexec"]}}
+{"http": {"ts":1484167928.239914,"uid":"CuRsfK1BjhmhzlDV45","id.orig_h":"192.168.138.158","id.orig_p":49190,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b2566564b3ba1a38e61c83957a7dbcd5","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167928.552572,"uid":"CuY6Oe1cwQi28U2sW","id.orig_h":"192.168.138.158","id.orig_p":49191,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?3a08b0be8322c244f5a1cb9c1057d941","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167928.866681,"uid":"C09cX52HiXhkT5Exa2","id.orig_h":"192.168.138.158","id.orig_p":49192,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?d71e0bd86db9587158745a986a4b3606","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167929.401205,"uid":"Cb5L1N1Ug32x6Oz4u3","id.orig_h":"192.168.138.158","id.orig_p":49193,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?34eaf8bd50d85d8c6baacb45f0a7b22e","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167929.705419,"uid":"CAhsP32ytUNibnJDX4","id.orig_h":"192.168.138.158","id.orig_p":49194,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?60dbe33b908e0086292196ef001816bc","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484167929.948512,"uid":"Cj4sal27hxyUSBNdG4","id.orig_h":"192.168.138.158","id.orig_p":49197,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?t=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FHBLd84QtlbCemYez2"],"orig_mime_types":["text/plain"],"resp_fuids":["FaaROi4lrbjm1FIpBa"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167930.004132,"uid":"CZiWcw1wYkyuuJ7dn2","id.orig_h":"192.168.138.158","id.orig_p":49196,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?51424ddd486ff06861fceed24e86b329","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"dns": {"ts":1484167929.445849,"uid":"ChakWs1TGdxYbia8H","id.orig_h":"192.168.138.158","id.orig_p":53571,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":15553,"query":"ip-addr.es","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["188.165.164.184"],"TTLs":[21599.0],"rejected":false}}
+{"dns": {"ts":1484167929.865123,"uid":"CVy9AO1a2S9g1ME1D","id.orig_h":"192.168.138.158","id.orig_p":61720,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":23625,"query":"runlove.us","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["204.152.254.221"],"TTLs":[14069.0],"rejected":false}}
+{"dns": {"ts":1484167930.106812,"uid":"CFdDBq1CQI2ScUH7c3","id.orig_h":"192.168.138.158","id.orig_p":50509,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":6088,"query":"kritischerkonsum.uni-koeln.de","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484167930.293382,"uid":"Co0gpI15MTwRMCkjN1","id.orig_h":"192.168.138.158","id.orig_p":56753,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":41589,"query":"comarksecurity.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["72.34.49.86"],"TTLs":[13888.0],"rejected":false}}
+{"http": {"ts":1484167930.378501,"uid":"CGfX4Z1TFOkZNVV1Ba","id.orig_h":"192.168.138.158","id.orig_p":49198,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?c=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FO2LTy2hFvukwN9GFi"],"orig_mime_types":["text/plain"],"resp_fuids":["FacbaL1rZYmMXVxSC7"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167932.531381,"uid":"CayZYU7BSUC868ND7","id.orig_h":"192.168.138.158","id.orig_p":49199,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?l=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["F8A2tO2f0GW4SwIiQ1"],"orig_mime_types":["text/plain"],"resp_fuids":["FXQqrf3jpgyu4s2Rka"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167932.764152,"uid":"CrLZ5CuSEtlRc7ZZ8","id.orig_h":"192.168.138.158","id.orig_p":49200,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?t=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":996,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["F8IiLaaCLRBk820Zh"],"orig_mime_types":["text/plain"],"resp_fuids":["FDNU5S22CAykjNpx9b"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167935.492514,"uid":"CvFLziqvgf8GKC8Y9","id.orig_h":"192.168.138.158","id.orig_p":49201,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FtWE1u3rLNc2AU2ZLh"],"orig_mime_types":["text/plain"],"resp_fuids":["FtIwWe2cQZ51hQ8Rv5"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167935.713955,"uid":"CLq0pehvmEuCqLi06","id.orig_h":"192.168.138.158","id.orig_p":49202,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":45662,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FB2jupcXqhiMRiem6"],"orig_mime_types":["text/plain"],"resp_fuids":["FVDw1t7CSEasxpe33"],"resp_mime_types":["image/png"]}}
+{"dns": {"ts":1484167930.866489,"uid":"Cu87p4iMM0QdpiTp6","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"http": {"ts":1484167941.875595,"uid":"CTWWO71KlU9DRrmnp7","id.orig_h":"192.168.138.158","id.orig_p":49203,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?f=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FpJNmm2r9zrMR9EgVg"],"orig_mime_types":["text/plain"],"resp_fuids":["FRwLTg9W436fghxr5"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167942.133849,"uid":"CcQP5R1PG4q2f1yT8","id.orig_h":"192.168.138.158","id.orig_p":49204,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FOy5gi3M2w8XWvNYqc"],"orig_mime_types":["text/plain"],"resp_fuids":["F4zMeEvy9RIBYHdt3"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167929.718969,"uid":"CU1z6c1RbgzgRhb2E3","id.orig_h":"192.168.138.158","id.orig_p":49195,"id.resp_h":"188.165.164.184","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ip-addr.es","uri":"/","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"tags":[]}}
+{"dns": {"ts":1484167968.175518,"uid":"CogghzajDK9MdvUb3","id.orig_h":"192.168.138.158","id.orig_p":50329,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":5810,"query":"7oqnsnzwwnm6zb7y.gigapaysun.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["95.163.121.204"],"TTLs":[14277.0],"rejected":false}}
+{"http": {"ts":1484167968.381714,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":3289,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FkXWFH1GnHj99Fx299"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167971.040006,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":4492,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJCJeM1R4x1H4y42dg"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484167972.844911,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/it.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":552,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fvu2eI2KDzXQvkD2Dh"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167972.844675,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/us.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":825,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FuSbCP13SdEpnVUFn"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167973.033914,"uid":"Ch2hGO1LrumL0QNPG","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":240,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FfcAU83ia8gfjLsWOk"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167973.033427,"uid":"CHwEGq1paXl7IqvzD3","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/picture.php?k=11iqmfg\u0026b7f2a994c3eaaf014608b272c46cf764","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":1823,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FfBfWC3nUGssCHBsL2"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167973.033645,"uid":"CzOF6l2DOLLYUZwvG4","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/es.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":634,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fil6QO3GWNnrBAiUf"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167973.049027,"uid":"Cbhgaw1IVL6NGqHpn2","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/de.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":534,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F4cZLM1Rfj48wYg1Pb"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167974.613828,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/fr.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":694,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FpjJ2mpIuKnU39Gve"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167974.82954,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":242,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F9K2Bp4ET8NJaBib48"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167974.864613,"uid":"Ch2hGO1LrumL0QNPG","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":239,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FY7LMa40QDG4zeNP5j"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167974.96311,"uid":"CHwEGq1paXl7IqvzD3","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":237,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FmYeOPLgRfGvDw3hj"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167977.095491,"uid":"CzOF6l2DOLLYUZwvG4","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/favicon.ico","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":318,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FUi3FI31tCwo4WgXS3"],"resp_mime_types":["image/x-icon"]}}
+{"http": {"ts":1484167982.036116,"uid":"Cbhgaw1IVL6NGqHpn2","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"POST","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":14,"response_body_len":14641,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FC94vR3f72342iAX92"],"orig_mime_types":["text/plain"],"resp_fuids":["Fk4Uzu2jQqXKUB00d5"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484167984.773279,"uid":"CcaM7Z1MyBBX9E8EC","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":304,"status_msg":"Not Modified","tags":[]}}
+{"http": {"ts":1484167986.879607,"uid":"CEmhjI1h5mAzRpnLN1","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/bitcoin.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":5523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FPcM0x2VeblSWANUz6"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484167986.880042,"uid":"Ch2hGO1LrumL0QNPG","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/button_pay.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":727,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fvv1TH12c7JL4UQUa2"],"resp_mime_types":["image/png"]}}
+{"dns": {"ts":1484167995.474042,"uid":"CgRzK32SqF5wJvC0Y4","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"http": {"ts":1484167924.526122,"uid":"CLKLkp1z9ZWAE0eou","id.orig_h":"192.168.138.158","id.orig_p":49186,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":121635,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FrcnSsZqVzpjB9o3j"],"resp_mime_types":["text/html"]}}
+{"dns": {"ts":1484168010.822008,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168011.828379,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168014.836836,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168023.85904,"uid":"Cg7uac12cgFflf6Fp7","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168048.700291,"uid":"Co6RaRZQIXvwkvaT8","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484168049.069438,"uid":"CeVtdzaICCMxZFAY9","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fx1AH04QmVyAYM5kd9"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168049.769399,"uid":"CZOU9CQKfQzbTKGZ8","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F95sxB3DPck4oMGLmc"],"resp_mime_types":["application/x-shockwave-flash"]}}
+{"dns": {"ts":1484168049.492644,"uid":"CXiPrelEswy2Vy506","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"dns": {"ts":1484168049.492947,"uid":"CddYI711hltuYI1aE7","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484168050.884154,"uid":"CyAev4UQJHk5ECqp6","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FxyuCg3ehaL3Q3Jicf"]}}
+{"http": {"ts":1484168051.288742,"uid":"CpF3KK32282sEULktb","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FbsI7rQSYdyoN8osc"],"resp_mime_types":["application/x-dosexec"]}}
+{"http": {"ts":1484168053.472796,"uid":"CTUxsD4ZOi3CcOT5Tc","id.orig_h":"192.168.138.158","id.orig_p":49190,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b2566564b3ba1a38e61c83957a7dbcd5","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484168053.7841,"uid":"CYsXRm4a452wUqMdpe","id.orig_h":"192.168.138.158","id.orig_p":49191,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?3a08b0be8322c244f5a1cb9c1057d941","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484168054.099112,"uid":"CQclu54qLZ704kkF4g","id.orig_h":"192.168.138.158","id.orig_p":49192,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?d71e0bd86db9587158745a986a4b3606","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484168054.633374,"uid":"CCOkcA3TQkLzkoUtVb","id.orig_h":"192.168.138.158","id.orig_p":49193,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?34eaf8bd50d85d8c6baacb45f0a7b22e","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"http": {"ts":1484168054.936005,"uid":"CXj9s84H83bRMeXVse","id.orig_h":"192.168.138.158","id.orig_p":49194,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?60dbe33b908e0086292196ef001816bc","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"dns": {"ts":1484168054.677804,"uid":"C0zadr4MkQXXg3R6ad","id.orig_h":"192.168.138.158","id.orig_p":53571,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":15553,"query":"ip-addr.es","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["188.165.164.184"],"TTLs":[21599.0],"rejected":false}}
+{"dns": {"ts":1484168055.095134,"uid":"ChdTCB23ylNPAWtcHj","id.orig_h":"192.168.138.158","id.orig_p":61720,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":23625,"query":"runlove.us","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["204.152.254.221"],"TTLs":[14069.0],"rejected":false}}
+{"http": {"ts":1484168055.178325,"uid":"C48TgT2oKyquocJlgi","id.orig_h":"192.168.138.158","id.orig_p":49197,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?t=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["Fn5ziO3r1iNx37j0Ml"],"orig_mime_types":["text/plain"],"resp_fuids":["Fqi2yO1cKqX6xLdMEe"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168055.233385,"uid":"C7A9tv3exoi6fTWTbl","id.orig_h":"192.168.138.158","id.orig_p":49196,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?51424ddd486ff06861fceed24e86b329","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":200,"status_msg":"OK","tags":[]}}
+{"dns": {"ts":1484168055.336209,"uid":"CQgJw93u2weQeXL7ch","id.orig_h":"192.168.138.158","id.orig_p":50509,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":6088,"query":"kritischerkonsum.uni-koeln.de","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168055.522671,"uid":"CcLbIe3Hv8tmqEXAql","id.orig_h":"192.168.138.158","id.orig_p":56753,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":41589,"query":"comarksecurity.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["72.34.49.86"],"TTLs":[13888.0],"rejected":false}}
+{"http": {"ts":1484168055.607448,"uid":"CUlrzm2SQPBB5osl6k","id.orig_h":"192.168.138.158","id.orig_p":49198,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?c=cdcnw7cfz43rmtg","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":134,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["F4QJql3CmvlqjPZ62c"],"orig_mime_types":["text/plain"],"resp_fuids":["FvG9Y93qeCBMqZaxl2"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484168057.760379,"uid":"CdUJwG2Df90m0Y7OSi","id.orig_h":"192.168.138.158","id.orig_p":49199,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?l=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["Fh9CoH303MQ3vTRjB"],"orig_mime_types":["text/plain"],"resp_fuids":["F9iisA25ZMf02F0vS5"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168057.992988,"uid":"CRAvCZ2ozjOE5ZgU9d","id.orig_h":"192.168.138.158","id.orig_p":49200,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?t=8r1gf1b2t1kuq42","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":96,"response_body_len":996,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FslJp33fnKpZkHE8y2"],"orig_mime_types":["text/plain"],"resp_fuids":["FJ8xAA0UPebOG5gAc"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484168060.720375,"uid":"CXT1m84PFOVKcQCote","id.orig_h":"192.168.138.158","id.orig_p":49201,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FOsK283Q71ZIaQFli"],"orig_mime_types":["text/plain"],"resp_fuids":["FqXoIQ1Wo0FMhwXkm"],"resp_mime_types":["text/html"]}}
+{"dns": {"ts":1484168050.914926,"uid":"CopjWUPbl0jgVvxuf","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"http": {"ts":1484168060.941201,"uid":"C759hO3qWwA0XQqi4g","id.orig_h":"192.168.138.158","id.orig_p":49202,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=mfymi71rapdzk","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":162,"response_body_len":45662,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FruxXL3ovqxbl9ZEt1"],"orig_mime_types":["text/plain"],"resp_fuids":["FXITBL1jZa8vBZWhOd"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168067.100666,"uid":"C8Np8A35YX2amygMVb","id.orig_h":"192.168.138.158","id.orig_p":49203,"id.resp_h":"204.152.254.221","id.resp_p":80,"trans_depth":1,"method":"POST","host":"runlove.us","uri":"/wp-content/themes/twentyfifteen/img5.php?f=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":357,"status_code":404,"status_msg":"Not Found","tags":[],"orig_fuids":["FYxhwp2IpRU8tG2ED1"],"orig_mime_types":["text/plain"],"resp_fuids":["F7VCjtX86bUyVy3R2"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168067.360726,"uid":"CI33ir4V9AqRYIQDSc","id.orig_h":"192.168.138.158","id.orig_p":49204,"id.resp_h":"72.34.49.86","id.resp_p":80,"trans_depth":1,"method":"POST","host":"comarksecurity.com","uri":"/wp-content/themes/grizzly/img5.php?u=ka6nnuvccqlw9","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":110,"response_body_len":14,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FSMLvV1wZHnHEWiRg6"],"orig_mime_types":["text/plain"],"resp_fuids":["FbdBIa2cOLEETy8ZM6"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484168054.949144,"uid":"CpYseO3TyKre605q5h","id.orig_h":"192.168.138.158","id.orig_p":49195,"id.resp_h":"188.165.164.184","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ip-addr.es","uri":"/","user_agent":"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"tags":[]}}
+{"dns": {"ts":1484168093.401627,"uid":"CQC8Pm4Qo5KRLMYqpe","id.orig_h":"192.168.138.158","id.orig_p":50329,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":5810,"query":"7oqnsnzwwnm6zb7y.gigapaysun.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["95.163.121.204"],"TTLs":[14277.0],"rejected":false}}
+{"http": {"ts":1484168093.608251,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":3289,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FOov1rV6rL28n8qy1"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168096.264793,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":4492,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FZjJMI3o09BdFRIgU7"],"resp_mime_types":["text/plain"]}}
+{"http": {"ts":1484168098.07042,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/it.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":552,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FEoclLaLEjZvuZZt9"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168098.069604,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/us.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":825,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FelBi52nX055gNTqoh"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168098.259569,"uid":"CuQjtm2b1ZTs5e0AHj","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":240,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F39o293n7WwhocflC7"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168098.259081,"uid":"CrtAyZ2wczgi6YrAch","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/picture.php?k=11iqmfg\u0026b7f2a994c3eaaf014608b272c46cf764","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":1823,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FB79Dm1t9dSUOI5eY6"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168098.259316,"uid":"CBmCBG2XG9D8KFerSi","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/es.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":634,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FJ14zE3SRYHfEVUBK6"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168098.274642,"uid":"C4PRKe3tPbLrANnoql","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":1,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/de.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":534,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fh4WuDHfRrX302wDj"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168099.839755,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/flags/fr.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":694,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["F8viU44xUigTtf4F4"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168100.055845,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rt.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":242,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FlQf7j3HXl7ZvoKcwe"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168100.091007,"uid":"CuQjtm2b1ZTs5e0AHj","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/lb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":239,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FqRNLE44JylF8wuffj"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168100.188585,"uid":"CrtAyZ2wczgi6YrAch","id.orig_h":"192.168.138.158","id.orig_p":49208,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/rb.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":237,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FAq3rD1EezTPYh1XVi"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168102.320148,"uid":"CBmCBG2XG9D8KFerSi","id.orig_h":"192.168.138.158","id.orig_p":49207,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/favicon.ico","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":318,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FsRCRQ1rW7SviqZ7rf"],"resp_mime_types":["image/x-icon"]}}
+{"dns": {"ts":1484168094.52272,"uid":"Chr2FK36PUE9wFhItb","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"http": {"ts":1484168107.260553,"uid":"C4PRKe3tPbLrANnoql","id.orig_h":"192.168.138.158","id.orig_p":49209,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":2,"method":"POST","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/11iQmfg","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":14,"response_body_len":14641,"status_code":200,"status_msg":"OK","tags":[],"orig_fuids":["FoLJFr1bniaMScBzXa"],"orig_mime_types":["text/plain"],"resp_fuids":["FMKGGw1xJPXlLO2r2"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168109.998211,"uid":"CZdkwD4Kbxc8UZOD6k","id.orig_h":"192.168.138.158","id.orig_p":49206,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/style.css","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":0,"status_code":304,"status_msg":"Not Modified","tags":[]}}
+{"http": {"ts":1484168112.105378,"uid":"CsUjA541poEzvhMfuf","id.orig_h":"192.168.138.158","id.orig_p":49205,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":4,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/bitcoin.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":5523,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FGcm94EWzm8st4LQj"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168112.106225,"uid":"CuQjtm2b1ZTs5e0AHj","id.orig_h":"192.168.138.158","id.orig_p":49210,"id.resp_h":"95.163.121.204","id.resp_p":80,"trans_depth":3,"method":"GET","host":"7oqnsnzwwnm6zb7y.gigapaysun.com","uri":"/img/button_pay.png","referrer":"http://7oqnsnzwwnm6zb7y.gigapaysun.com/11iQmfg","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":727,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FpDS0Lu22MwpGf8ac"],"resp_mime_types":["image/png"]}}
+{"http": {"ts":1484168049.78326,"uid":"CJ8TuiTOmndD9weBa","id.orig_h":"192.168.138.158","id.orig_p":49186,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":121635,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FXdaxF1PjrvmYbJfLg"],"resp_mime_types":["text/html"]}}
+{"dns": {"ts":1484168130.768775,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168131.774252,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168134.783524,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168143.802793,"uid":"CSpFkT2sFGZoEEZ3gi","id.orig_h":"192.168.66.1","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false}}
+{"dns": {"ts":1484168173.926757,"uid":"CVf8zv3sBOdNwWTrbl","id.orig_h":"192.168.138.158","id.orig_p":60078,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":18350,"query":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"dns": {"ts":1484168174.717258,"uid":"CCbfqc3ox2hz3kqRSj","id.orig_h":"192.168.138.158","id.orig_p":65315,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":27248,"query":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"dns": {"ts":1484168174.717487,"uid":"CSHdCU2z9CrpPiMuXk","id.orig_h":"192.168.138.158","id.orig_p":50683,"id.resp_h":"192.168.138.2","id.resp_p":53,"proto":"udp","trans_id":62139,"query":"r03afd2.c3008e.xc07r.b0f.a39.h7f0fa5eu.vb8fbl.e8mfzdgrf7g0.groupprograms.in","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["62.75.195.236"],"TTLs":[29.0],"rejected":false}}
+{"http": {"ts":1484168174.29515,"uid":"CHg4AB2DzmEvPnlJoi","id.orig_h":"192.168.138.158","id.orig_p":49184,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in","uri":"/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":560,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FbYOC42kRE93hxmUOb"],"resp_mime_types":["text/html"]}}
+{"http": {"ts":1484168174.994035,"uid":"C3MrJz2uc4HxL7lCZg","id.orig_h":"192.168.138.158","id.orig_p":49185,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"ubb67.3c147o.u806a4.w07d919.o5f.f1.b80w.r0faf9.e8mfzdgrf7g0.groupprograms.in","uri":"/","referrer":"http://va872g.g90e1h.b8.642b63u.j985a2.v33e.37.pa269cc.e8mfzdgrf7g0.groupprograms.in/?285a4d4e4e5a4d4d4649584c5d43064b4745","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":8973,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fz99Ml4Hgbcn29QSMa"],"resp_mime_types":["application/x-shockwave-flash"]}}
+{"http": {"ts":1484168176.096419,"uid":"CZerqX2fu5NcfhGiZj","id.orig_h":"192.168.138.158","id.orig_p":49188,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/aa25f5fe2875e3d0a244e6969e589cc4","request_body_len":0,"response_body_len":861,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["Fixn8N2eNk8O3WwOti"]}}
+{"http": {"ts":1484168176.500568,"uid":"C7fLUD28ahyzAuVnAl","id.orig_h":"192.168.138.158","id.orig_p":49189,"id.resp_h":"62.75.195.236","id.resp_p":80,"trans_depth":1,"method":"GET","host":"62.75.195.236","uri":"/?b514ee6f0fe486009a6d83b035a4c0bd","user_agent":"Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)","request_body_len":0,"response_body_len":221184,"status_code":200,"status_msg":"OK
<TRUNCATED>
[41/50] [abbrv] metron git commit: METRON-1447 Heap Size Not Set
Correctly by MPack for ES 5.x (nickwallen) closes apache/metron#927
Posted by rm...@apache.org.
METRON-1447 Heap Size Not Set Correctly by MPack for ES 5.x (nickwallen) closes apache/metron#927
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/31f3aa8c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/31f3aa8c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/31f3aa8c
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 31f3aa8cb0f4c3821c264e375aa14eda207adcb0
Parents: 38b7d5e
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Feb 7 14:50:06 2018 -0500
Committer: nickallen <ni...@apache.org>
Committed: Wed Feb 7 14:50:06 2018 -0500
----------------------------------------------------------------------
.../5.6.2/configuration/elastic-jvm-options.xml | 144 +++++++++++++++++++
.../5.6.2/configuration/elastic-sysconfig.xml | 12 +-
.../ELASTICSEARCH/5.6.2/metainfo.xml | 1 +
.../5.6.2/package/scripts/elastic_commands.py | 13 ++
.../5.6.2/package/scripts/params.py | 3 +
5 files changed, 163 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
new file mode 100644
index 0000000..5c6aaca
--- /dev/null
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-jvm-options.xml
@@ -0,0 +1,144 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <property>
+ <name>heap_size</name>
+ <value>512m</value>
+ <description>JVM heap size</description>
+ </property>
+ <property>
+ <name>content</name>
+ <description>The jinja template for the Elasticsearch JVM options file.</description>
+ <value>
+## JVM configuration
+
+################################################################
+## IMPORTANT: JVM heap size
+################################################################
+##
+## You should always set the min and max JVM heap
+## size to the same value. For example, to set
+## the heap to 4 GB, set:
+##
+## -Xms4g
+## -Xmx4g
+##
+## See https://www.elastic.co/guide/en/elasticsearch/reference/current/heap-size.html
+## for more information
+##
+################################################################
+
+# Xms represents the initial size of total heap space
+# Xmx represents the maximum size of total heap space
+
+-Xms{{heap_size}}
+-Xmx{{heap_size}}
+
+################################################################
+## Expert settings
+################################################################
+##
+## All settings below this section are considered
+## expert settings. Don't tamper with them unless
+## you understand what you are doing
+##
+################################################################
+
+## GC configuration
+-XX:+UseConcMarkSweepGC
+-XX:CMSInitiatingOccupancyFraction=75
+-XX:+UseCMSInitiatingOccupancyOnly
+
+## optimizations
+
+# pre-touch memory pages used by the JVM during initialization
+-XX:+AlwaysPreTouch
+
+## basic
+
+# force the server VM (remove on 32-bit client JVMs)
+-server
+
+# explicitly set the stack size (reduce to 320k on 32-bit client JVMs)
+-Xss1m
+
+# set to headless, just in case
+-Djava.awt.headless=true
+
+# ensure UTF-8 encoding by default (e.g. filenames)
+-Dfile.encoding=UTF-8
+
+# use our provided JNA always versus the system one
+-Djna.nosys=true
+
+# use old-style file permissions on JDK9
+-Djdk.io.permissionsUseCanonicalPath=true
+
+# flags to configure Netty
+-Dio.netty.noUnsafe=true
+-Dio.netty.noKeySetOptimization=true
+-Dio.netty.recycler.maxCapacityPerThread=0
+
+# log4j 2
+-Dlog4j.shutdownHookEnabled=false
+-Dlog4j2.disable.jmx=true
+-Dlog4j.skipJansi=true
+
+## heap dumps
+
+# generate a heap dump when an allocation from the Java heap fails
+# heap dumps are created in the working directory of the JVM
+-XX:+HeapDumpOnOutOfMemoryError
+
+# specify an alternative path for heap dumps
+# ensure the directory exists and has sufficient space
+#-XX:HeapDumpPath=${heap.dump.path}
+
+## GC logging
+
+#-XX:+PrintGCDetails
+#-XX:+PrintGCTimeStamps
+#-XX:+PrintGCDateStamps
+#-XX:+PrintClassHistogram
+#-XX:+PrintTenuringDistribution
+#-XX:+PrintGCApplicationStoppedTime
+
+# log GC status to a file with time stamps
+# ensure the directory exists
+#-Xloggc:${loggc}
+
+# By default, the GC log file will not rotate.
+# By uncommenting the lines below, the GC log file
+# will be rotated every 128MB at most 32 times.
+#-XX:+UseGCLogFileRotation
+#-XX:NumberOfGCLogFiles=32
+#-XX:GCLogFileSize=128M
+
+# Elasticsearch 5.0.0 will throw an exception on unquoted field names in JSON.
+# If documents were already indexed with unquoted fields in a previous version
+# of Elasticsearch, some operations may throw errors.
+#
+# WARNING: This option will be removed in Elasticsearch 6.0.0 and is provided
+# only for migration purposes.
+#-Delasticsearch.json.allow_unquoted_field_names=true
+ </value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
index cb069b8..ea6ca38 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/configuration/elastic-sysconfig.xml
@@ -40,11 +40,6 @@
<description>Elasticsearch Configuration Directory</description>
</property>
<property>
- <name>heap_size</name>
- <value>512m</value>
- <description>Heap size</description>
- </property>
- <property>
<name>max_open_files</name>
<value>65536</value>
<description>Maximum number of open files</description>
@@ -92,11 +87,8 @@ PID_DIR={{pid_dir}}
# JAVA_HOME must be provided here for OS that use systemd service launch
JAVA_HOME={{java64_home}}
-# Additional Java OPTS
-ES_JAVA_OPTS="-verbose:gc -Xloggc:{{log_dir}}/elasticsearch_gc.log -XX:-CMSConcurrentMTEnabled \
--XX:+PrintGCDateStamps -XX:+PrintGCDetails -XX:+PrintGCTimeStamps \
--XX:ErrorFile={{log_dir}}/elasticsearch_err.log -XX:ParallelGCThreads=8 \
--Xms{{heap_size}} -Xmx{{heap_size}}"
+# Additional Java options - now preferential to use 'jvm.options' file instead
+ES_JAVA_OPTS=""
# https://www.elastic.co/guide/en/elasticsearch/reference/5.6/_memory_lock_check.html
MAX_LOCKED_MEMORY=unlimited
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
index 0943eec..47abb45 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/metainfo.xml
@@ -83,6 +83,7 @@
<config-type>elastic-site</config-type>
<config-type>elastic-sysconfig</config-type>
<config-type>elastic-systemd</config-type>
+ <config-type>elastic-jvm-options</config-type>
</configuration-dependencies>
<restartRequiredAfterChange>true</restartRequiredAfterChange>
<quickLinksConfigurations>
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
index afbaff2..618d10a 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/elastic_commands.py
@@ -190,6 +190,17 @@ def create_elastic_pam_limits(params):
owner="root",
group="root")
+def create_elastic_jvm_options(params):
+ """
+ Defines the jvm.options file used to specify JVM options.
+ """
+ path = "{0}/jvm.options".format(params.conf_dir)
+ Logger.info("Creating Elasticsearch JVM Options; file={0}".format(path))
+ File(path,
+ content=InlineTemplate(params.jvm_options_template),
+ owner=params.elastic_user,
+ group=params.elastic_group)
+
def get_data_directories(params):
"""
Returns the directories to use for storing Elasticsearch data.
@@ -225,6 +236,7 @@ def configure_master():
create_elastic_site(params, "elasticsearch.master.yaml.j2")
create_elastic_config(params)
create_elastic_pam_limits(params)
+ create_elastic_jvm_options(params)
if is_systemd_running():
configure_systemd(params)
@@ -249,5 +261,6 @@ def configure_slave():
create_elastic_site(params, "elasticsearch.slave.yaml.j2")
create_elastic_config(params)
create_elastic_pam_limits(params)
+ create_elastic_jvm_options(params)
if is_systemd_running():
configure_systemd(params)
http://git-wip-us.apache.org/repos/asf/metron/blob/31f3aa8c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
index 0399c60..24f2306 100755
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/ELASTICSEARCH/5.6.2/package/scripts/params.py
@@ -103,3 +103,6 @@ systemd_parent_dir = '/etc/systemd/system/'
systemd_elasticsearch_dir = systemd_parent_dir + 'elasticsearch.service.d/'
systemd_override_file = systemd_elasticsearch_dir + 'override.conf'
systemd_override_template = config['configurations']['elastic-systemd']['content']
+
+heap_size = config['configurations']['elastic-jvm-options']['heap_size']
+jvm_options_template = config['configurations']['elastic-jvm-options']['content']
[42/50] [abbrv] metron git commit: METRON-1273: Website documentation
link should point to the current site-book (JonZeolla via mmiklavc) closes
apache/metron#812
Posted by rm...@apache.org.
METRON-1273: Website documentation link should point to the current site-book (JonZeolla via mmiklavc) closes apache/metron#812
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e5f24f7b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e5f24f7b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e5f24f7b
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: e5f24f7bdce6a119adb8be543752d458f721d48d
Parents: 31f3aa8
Author: JonZeolla <ze...@gmail.com>
Authored: Wed Feb 7 13:58:11 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Wed Feb 7 13:58:11 2018 -0700
----------------------------------------------------------------------
site/documentation/index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/e5f24f7b/site/documentation/index.md
----------------------------------------------------------------------
diff --git a/site/documentation/index.md b/site/documentation/index.md
index 6b38ed6..e44953c 100644
--- a/site/documentation/index.md
+++ b/site/documentation/index.md
@@ -67,7 +67,7 @@ title: Apache Metron Documentation
</div>
<div class="content-960 hover-btn text-center">
<p> Apache Metron documentation provides information on quickly getting started with Metron, performing a full installation, adding additional data sources, and using Metron to triage alerts. Metron documentation is currently a work in progress. Please check back again later as we continue to grow this documentation set.</p>
- <a class="button-default" href=" https://cwiki.apache.org/confluence/display/METRON/Documentation" target="_blank">LEARN MORE</a>
+ <a class="button-default" href="https://metron.apache.org/current-book/index.html" target="_blank">LEARN MORE</a>
</div>
</section>
[22/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/sensor-stubs/files/snort.out
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/sensor-stubs/files/snort.out b/metron-deployment/ansible/roles/sensor-stubs/files/snort.out
new file mode 100644
index 0000000..32f9a53
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/files/snort.out
@@ -0,0 +1,27404 @@
+01/11/17-20:49:18.107168 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E396BFC,0x56900BB6,,0x1000,64,10,23403,76,77824,,,,
+01/11/17-20:49:18.107195 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900BB6,0x1E396C14,,0x1F13,64,16,3894,52,53248,,,,
+01/11/17-20:49:18.107396 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396C14,0x56900BCE,,0xFFF,64,8,40177,52,53248,,,,
+01/11/17-20:49:18.107510 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E396C14,0x56900BCE,,0x1000,64,10,53179,204,208896,,,,
+01/11/17-20:49:18.109884 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CAC,0x56900C16,,0xFFD,64,8,30920,52,53248,,,,
+01/11/17-20:49:18.132533 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CAC,0x56900C7E,,0xFFC,64,8,44619,52,53248,,,,
+01/11/17-20:49:18.132540 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CAC,0x56900CAE,,0xFFE,64,8,55021,52,53248,,,,
+01/11/17-20:49:18.132751 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E396CAC,0x56900CAE,,0x1000,64,10,24370,76,77824,,,,
+01/11/17-20:49:18.154679 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396CC4,0x56900CAE,,0x1000,64,10,50877,92,94208,,,,
+01/11/17-20:49:18.154802 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900CAE,0x1E396CEC,,0x1F13,64,16,3899,52,53248,,,,
+01/11/17-20:49:18.155051 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CEC,0x56900CC6,,0xFFF,64,8,6931,52,53248,,,,
+01/11/17-20:49:18.155074 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E396CEC,0x56900CC6,,0x1000,64,10,50478,116,118784,,,,
+01/11/17-20:49:18.165247 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396D2C,0x56900CF6,,0xFFE,64,8,57108,52,53248,,,,
+01/11/17-20:49:18.315958 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D2C,0x56900CF6,,0x1000,64,10,31240,92,94208,,,,
+01/11/17-20:49:18.315973 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D2C,0x56900CF6,,0x1000,64,8,51328,92,94208,,,,
+01/11/17-20:49:18.315979 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4E,***A****,0x56900CF6,0x1E396D54,,0x1F13,64,16,3902,64,65536,,,,
+01/11/17-20:49:18.317192 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396D54,0x56900D0E,,0xFFF,64,8,8422,52,53248,,,,
+01/11/17-20:49:18.317376 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D54,0x56900D0E,,0x1000,64,10,26873,92,94208,,,,
+01/11/17-20:49:18.317729 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396D7C,0x56900D26,,0xFFF,64,8,63956,52,53248,,,,
+01/11/17-20:49:18.318013 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D7C,0x56900D26,,0x1000,64,10,21452,92,94208,,,,
+01/11/17-20:49:18.318328 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396DA4,0x56900D3E,,0xFFF,64,8,64316,52,53248,,,,
+01/11/17-20:49:18.323894 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396DA4,0x56900D3E,,0x1000,64,10,49851,92,94208,,,,
+01/11/17-20:49:18.324357 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396DCC,0x56900D56,,0xFFF,64,8,54379,52,53248,,,,
+01/11/17-20:49:18.325050 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E396DCC,0x56900D56,,0x1000,64,10,63932,1500,225300,,,,
+01/11/17-20:49:18.325060 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E397374,0x56900D56,,0x1000,64,10,34229,1060,36880,,,,
+01/11/17-20:49:18.325065 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E397764,,0x1F13,64,16,3907,52,53248,,,,
+01/11/17-20:49:18.325419 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E397764,0x56900D56,,0x1000,64,10,59177,1500,225300,,,,
+01/11/17-20:49:18.325425 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E397D0C,0x56900D56,,0x1000,64,10,49253,1500,225300,,,,
+01/11/17-20:49:18.325427 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E3982B4,,0x1F13,64,16,3908,52,53248,,,,
+01/11/17-20:49:18.325439 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4A,***AP***,0x1E3982B4,0x56900D56,,0x1000,64,10,12980,60,61440,,,,
+01/11/17-20:49:18.325798 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3982BC,0x56900D56,,0x1000,64,10,17183,1500,225300,,,,
+01/11/17-20:49:18.325803 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E398864,,0x1F13,64,16,3909,52,53248,,,,
+01/11/17-20:49:18.325813 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E398864,0x56900D56,,0x1000,64,10,10585,644,135176,,,,
+01/11/17-20:49:18.326167 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E398AB4,0x56900D56,,0x1000,64,10,64378,1500,225300,,,,
+01/11/17-20:49:18.326173 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E39905C,,0x1F13,64,16,3910,52,53248,,,,
+01/11/17-20:49:18.326183 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E39905C,0x56900D56,,0x1000,64,10,36369,900,135180,,,,
+01/11/17-20:49:18.326185 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3993AC,0x56900D56,,0x1000,64,10,42540,92,94208,,,,
+01/11/17-20:49:18.330409 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E3993D4,,0x1F13,64,16,3911,52,53248,,,,
+01/11/17-20:49:18.331204 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3993D4,0x56900D6E,,0xFFF,64,8,42136,52,53248,,,,
+01/11/17-20:49:18.331625 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3993D4,0x56900D6E,,0x1000,64,10,48207,1500,225300,,,,
+01/11/17-20:49:18.331641 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E39997C,0x56900D6E,,0x1000,64,10,21064,292,36868,,,,
+01/11/17-20:49:18.331645 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E399A6C,,0x1F13,64,16,3913,52,53248,,,,
+01/11/17-20:49:18.331923 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E399A6C,0x56900D6E,,0x1000,64,10,15105,1500,225300,,,,
+01/11/17-20:49:18.331938 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E39A014,0x56900D6E,,0x1000,64,10,1341,468,217092,,,,
+01/11/17-20:49:18.331941 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39A1B4,,0x1F13,64,16,3914,52,53248,,,,
+01/11/17-20:49:18.332270 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39A1B4,0x56900D6E,,0x1000,64,10,10751,1500,225300,,,,
+01/11/17-20:49:18.332284 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E39A75C,0x56900D6E,,0x1000,64,10,2115,164,167936,,,,
+01/11/17-20:49:18.332286 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39A7CC,,0x1F13,64,16,3915,52,53248,,,,
+01/11/17-20:49:18.332650 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39A7CC,0x56900D6E,,0x1000,64,10,10561,1500,225300,,,,
+01/11/17-20:49:18.332655 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E39AD74,0x56900D6E,,0x1000,64,10,42742,788,20492,,,,
+01/11/17-20:49:18.332658 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39B054,,0x1F13,64,16,3916,52,53248,,,,
+01/11/17-20:49:18.333034 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39B054,0x56900D6E,,0x1000,64,10,56541,1500,225300,,,,
+01/11/17-20:49:18.333048 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E39B5FC,0x56900D6E,,0x1000,64,10,25591,948,184332,,,,
+01/11/17-20:49:18.333052 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39B97C,,0x1F13,64,16,3917,52,53248,,,,
+01/11/17-20:49:18.333403 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39B97C,0x56900D86,,0xFFF,64,8,24503,52,53248,,,,
+01/11/17-20:49:18.333409 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39B97C,0x56900D86,,0x1000,64,10,63480,1500,225300,,,,
+01/11/17-20:49:18.333421 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E39BF24,0x56900D86,,0x1000,64,10,61701,964,200716,,,,
+01/11/17-20:49:18.333424 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D86,0x1E39C2B4,,0x1F13,64,16,3919,52,53248,,,,
+01/11/17-20:49:18.333816 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39C2B4,0x56900D86,,0x1000,64,10,25421,1500,225300,,,,
+01/11/17-20:49:18.333820 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E39C85C,0x56900D86,,0x1000,64,10,25178,756,249864,,,,
+01/11/17-20:49:18.333822 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D86,0x1E39CB1C,,0x1F13,64,16,3920,52,53248,,,,
+01/11/17-20:49:18.334060 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB1C,0x56900D9E,,0xFFF,64,8,59095,52,53248,,,,
+01/11/17-20:49:18.334219 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CB1C,0x56900D9E,,0x1000,64,10,61490,92,94208,,,,
+01/11/17-20:49:18.334426 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB44,0x56900DB6,,0xFFF,64,8,62611,52,53248,,,,
+01/11/17-20:49:18.334608 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CB44,0x56900DB6,,0x1000,64,10,33533,76,77824,,,,
+01/11/17-20:49:18.335128 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB5C,0x56900DFE,,0xFFD,64,8,44344,52,53248,,,,
+01/11/17-20:49:18.335229 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CB5C,0x56900DFE,,0x1000,64,10,65063,76,77824,,,,
+01/11/17-20:49:18.348745 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CB74,0x56900DFE,,0x1000,64,10,45295,92,94208,,,,
+01/11/17-20:49:18.348795 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900DFE,0x1E39CB9C,,0x1F13,64,16,3924,52,53248,,,,
+01/11/17-20:49:18.349013 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB9C,0x56900E16,,0xFFF,64,8,20224,52,53248,,,,
+01/11/17-20:49:18.349132 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E39CB9C,0x56900E16,,0x1000,64,10,37526,556,45064,,,,
+01/11/17-20:49:18.352001 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900E5E,,0xFFD,64,8,37910,52,53248,,,,
+01/11/17-20:49:18.383171 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900E96,,0xFFE,64,8,36703,52,53248,,,,
+01/11/17-20:49:18.441681 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900ECE,,0xFFE,64,8,55590,52,53248,,,,
+01/11/17-20:49:18.451874 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900F2E,,0xFFD,64,8,19749,52,53248,,,,
+01/11/17-20:49:18.452037 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CD94,0x56900F2E,,0x1000,64,10,168,76,77824,,,,
+01/11/17-20:49:18.491519 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900F2E,0x1E39CDAC,,0x1F13,64,16,3930,52,53248,,,,
+01/11/17-20:49:28.469092 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CDAC,0x56900F2E,,0x1000,64,10,30215,76,77824,,,,
+01/11/17-20:49:28.469122 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900F2E,0x1E39CDC4,,0x1F13,64,16,3931,52,53248,,,,
+01/11/17-20:49:28.469442 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CDC4,0x56900F46,,0xFFF,64,8,17853,52,53248,,,,
+01/11/17-20:49:28.469543 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E39CDC4,0x56900F46,,0x1000,64,10,25675,204,208896,,,,
+01/11/17-20:49:28.470972 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE5C,0x56900F8E,,0xFFD,64,8,39640,52,53248,,,,
+01/11/17-20:49:28.482209 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE5C,0x56900FE6,,0xFFD,64,8,45105,52,53248,,,,
+01/11/17-20:49:28.482287 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE5C,0x56901016,,0xFFE,64,8,44659,52,53248,,,,
+01/11/17-20:49:28.482352 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CE5C,0x56901016,,0x1000,64,10,64668,76,77824,,,,
+01/11/17-20:49:28.500940 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CE74,0x56901016,,0x1000,64,10,18360,92,94208,,,,
+01/11/17-20:49:28.501082 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901016,0x1E39CE9C,,0x1F13,64,16,3936,52,53248,,,,
+01/11/17-20:49:28.501262 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE9C,0x5690102E,,0xFFF,64,8,33893,52,53248,,,,
+01/11/17-20:49:28.501342 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E39CE9C,0x5690102E,,0x1000,64,10,50754,116,118784,,,,
+01/11/17-20:49:28.502358 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CEDC,0x5690105E,,0xFFE,64,8,42796,52,53248,,,,
+01/11/17-20:49:28.502487 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CEDC,0x5690105E,,0x1000,64,10,9919,92,94208,,,,
+01/11/17-20:49:28.512827 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF04,0x56901076,,0xFFF,64,8,49447,52,53248,,,,
+01/11/17-20:49:28.512959 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CF04,0x56901076,,0x1000,64,10,29403,92,94208,,,,
+01/11/17-20:49:28.513418 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF2C,0x5690108E,,0xFFF,64,8,19625,52,53248,,,,
+01/11/17-20:49:28.513717 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CF2C,0x5690108E,,0x1000,64,10,21710,92,94208,,,,
+01/11/17-20:49:28.514000 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF54,0x569010A6,,0xFFF,64,8,10409,52,53248,,,,
+01/11/17-20:49:28.519867 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CF54,0x569010A6,,0x1000,64,10,7830,92,94208,,,,
+01/11/17-20:49:28.520589 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF7C,0x569010BE,,0xFFF,64,8,6502,52,53248,,,,
+01/11/17-20:49:28.520731 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39CF7C,0x569010BE,,0x1000,64,10,51267,1500,225300,,,,
+01/11/17-20:49:28.520747 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E39D524,0x569010BE,,0x1000,64,10,15295,1060,36880,,,,
+01/11/17-20:49:28.520750 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39D914,,0x1F13,64,16,3943,52,53248,,,,
+01/11/17-20:49:28.521160 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39D914,0x569010BE,,0x1000,64,10,42479,1500,225300,,,,
+01/11/17-20:49:28.521166 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39DEBC,0x569010BE,,0x1000,64,10,65233,1500,225300,,,,
+01/11/17-20:49:28.521168 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39E464,,0x1F13,64,16,3944,52,53248,,,,
+01/11/17-20:49:28.521179 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4A,***AP***,0x1E39E464,0x569010BE,,0x1000,64,10,6499,60,61440,,,,
+01/11/17-20:49:28.521543 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39E46C,0x569010BE,,0x1000,64,10,36882,1500,225300,,,,
+01/11/17-20:49:28.521548 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39EA14,,0x1F13,64,16,3945,52,53248,,,,
+01/11/17-20:49:28.521559 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E39EA14,0x569010BE,,0x1000,64,10,49793,644,135176,,,,
+01/11/17-20:49:28.522079 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39EC64,0x569010BE,,0x1000,64,10,56708,1500,225300,,,,
+01/11/17-20:49:28.522084 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39F20C,,0x1F13,64,16,3946,52,53248,,,,
+01/11/17-20:49:28.522105 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E39F20C,0x569010BE,,0x1000,64,10,33698,900,135180,,,,
+01/11/17-20:49:28.522107 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39F55C,0x569010BE,,0x1000,64,10,15201,92,94208,,,,
+01/11/17-20:49:28.522148 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39F584,,0x1F13,64,16,3947,52,53248,,,,
+01/11/17-20:49:28.522763 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39F584,0x569010D6,,0xFFF,64,8,64832,52,53248,,,,
+01/11/17-20:49:28.522911 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39F584,0x569010D6,,0x1000,64,10,18426,1500,225300,,,,
+01/11/17-20:49:28.522933 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E39FB2C,0x569010D6,,0x1000,64,10,55360,292,36868,,,,
+01/11/17-20:49:28.522938 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E39FC1C,,0x1F13,64,16,3949,52,53248,,,,
+01/11/17-20:49:28.523205 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39FC1C,0x569010D6,,0x1000,64,10,4284,1500,225300,,,,
+01/11/17-20:49:28.523209 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E3A01C4,0x569010D6,,0x1000,64,10,12946,468,217092,,,,
+01/11/17-20:49:28.523211 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A0364,,0x1F13,64,16,3950,52,53248,,,,
+01/11/17-20:49:28.523456 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A0364,0x569010D6,,0x1000,64,10,7500,1500,225300,,,,
+01/11/17-20:49:28.523459 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E3A090C,0x569010D6,,0x1000,64,10,40144,164,167936,,,,
+01/11/17-20:49:28.523461 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A097C,,0x1F13,64,16,3951,52,53248,,,,
+01/11/17-20:49:28.523873 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A097C,0x569010D6,,0x1000,64,10,16560,1500,225300,,,,
+01/11/17-20:49:28.523877 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E3A0F24,0x569010D6,,0x1000,64,10,36010,788,20492,,,,
+01/11/17-20:49:28.523880 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A1204,,0x1F13,64,16,3952,52,53248,,,,
+01/11/17-20:49:28.524221 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A1204,0x569010D6,,0x1000,64,10,63499,1500,225300,,,,
+01/11/17-20:49:28.524225 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E3A17AC,0x569010D6,,0x1000,64,10,48918,948,184332,,,,
+01/11/17-20:49:28.524227 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A1B2C,,0x1F13,64,16,3953,52,53248,,,,
+01/11/17-20:49:28.524579 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A1B2C,0x569010D6,,0x1000,64,10,10124,1500,225300,,,,
+01/11/17-20:49:28.524582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E3A20D4,0x569010D6,,0x1000,64,10,60584,964,200716,,,,
+01/11/17-20:49:28.524585 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A2464,,0x1F13,64,16,3954,52,53248,,,,
+01/11/17-20:49:28.524889 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A2464,0x569010D6,,0x1000,64,10,30197,1500,225300,,,,
+01/11/17-20:49:28.524893 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E3A2A0C,0x569010D6,,0x1000,64,10,14240,756,249864,,,,
+01/11/17-20:49:28.524895 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A2CCC,,0x1F13,64,16,3955,52,53248,,,,
+01/11/17-20:49:28.525595 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2CCC,0x569010EE,,0xFFF,64,8,44823,52,53248,,,,
+01/11/17-20:49:28.525764 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2CCC,0x56901106,,0xFFF,64,8,60365,52,53248,,,,
+01/11/17-20:49:28.525893 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A2CCC,0x56901106,,0x1000,64,10,22857,92,94208,,,,
+01/11/17-20:49:28.526459 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2CF4,0x5690111E,,0xFFF,64,8,36935,52,53248,,,,
+01/11/17-20:49:28.526578 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2CF4,0x5690111E,,0x1000,64,10,1319,76,77824,,,,
+01/11/17-20:49:28.527102 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2D0C,0x56901166,,0xFFD,64,8,14691,52,53248,,,,
+01/11/17-20:49:28.527180 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2D0C,0x56901166,,0x1000,64,10,32628,76,77824,,,,
+01/11/17-20:49:28.540964 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A2D24,0x56901166,,0x1000,64,10,52141,92,94208,,,,
+01/11/17-20:49:28.541023 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901166,0x1E3A2D4C,,0x1F13,64,16,3960,52,53248,,,,
+01/11/17-20:49:28.541279 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2D4C,0x5690117E,,0xFFF,64,8,49024,52,53248,,,,
+01/11/17-20:49:28.541440 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E3A2D4C,0x5690117E,,0x1000,64,10,64071,556,45064,,,,
+01/11/17-20:49:28.542767 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x569011C6,,0xFFD,64,8,20305,52,53248,,,,
+01/11/17-20:49:28.568085 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x569011FE,,0xFFE,64,8,35412,52,53248,,,,
+01/11/17-20:49:28.618582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x56901236,,0xFFE,64,8,60632,52,53248,,,,
+01/11/17-20:49:28.626337 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x56901296,,0xFFD,64,8,20453,52,53248,,,,
+01/11/17-20:49:28.626346 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2F44,0x56901296,,0x1000,64,10,3210,76,77824,,,,
+01/11/17-20:49:28.671946 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901296,0x1E3A2F5C,,0x1F13,64,16,3966,52,53248,,,,
+01/11/17-20:49:38.644842 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2F5C,0x56901296,,0x1000,64,10,35540,76,77824,,,,
+01/11/17-20:49:38.644865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901296,0x1E3A2F74,,0x1F13,64,16,3967,52,53248,,,,
+01/11/17-20:49:38.645301 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F74,0x569012AE,,0xFFF,64,8,38915,52,53248,,,,
+01/11/17-20:49:38.645410 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E3A2F74,0x569012AE,,0x1000,64,10,49946,204,208896,,,,
+01/11/17-20:49:38.655935 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A300C,0x569012F6,,0xFFD,64,8,49982,52,53248,,,,
+01/11/17-20:49:38.678474 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A300C,0x5690135E,,0xFFC,64,8,32965,52,53248,,,,
+01/11/17-20:49:38.678482 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A300C,0x5690138E,,0xFFE,64,8,24815,52,53248,,,,
+01/11/17-20:49:38.678626 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A300C,0x5690138E,,0x1000,64,10,26150,76,77824,,,,
+01/11/17-20:49:38.698531 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A3024,0x5690138E,,0x1000,64,10,57225,92,94208,,,,
+01/11/17-20:49:38.698676 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690138E,0x1E3A304C,,0x1F13,64,16,3972,52,53248,,,,
+01/11/17-20:49:38.698875 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A304C,0x569013A6,,0xFFF,64,8,22084,52,53248,,,,
+01/11/17-20:49:38.698882 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E3A304C,0x569013A6,,0x1000,64,10,64622,116,118784,,,,
+01/11/17-20:49:38.699881 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A308C,0x569013D6,,0xFFE,64,8,44450,52,53248,,,,
+01/11/17-20:49:38.700138 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A308C,0x569013D6,,0x1000,64,10,42900,92,94208,,,,
+01/11/17-20:49:38.710613 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A30B4,0x569013EE,,0xFFF,64,8,64776,52,53248,,,,
+01/11/17-20:49:38.710768 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A30B4,0x569013EE,,0x1000,64,10,19867,92,94208,,,,
+01/11/17-20:49:38.711060 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A30DC,0x56901406,,0xFFF,64,8,52782,52,53248,,,,
+01/11/17-20:49:38.711262 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A30DC,0x56901406,,0x1000,64,10,10692,92,94208,,,,
+01/11/17-20:49:38.711505 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A3104,0x5690141E,,0xFFF,64,8,8814,52,53248,,,,
+01/11/17-20:49:38.717620 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A3104,0x5690141E,,0x1000,64,10,62092,92,94208,,,,
+01/11/17-20:49:38.718189 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A312C,0x56901436,,0xFFF,64,8,42105,52,53248,,,,
+01/11/17-20:49:38.718603 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A312C,0x56901436,,0x1000,64,10,64933,1500,225300,,,,
+01/11/17-20:49:38.718608 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E3A36D4,0x56901436,,0x1000,64,10,26575,1060,36880,,,,
+01/11/17-20:49:38.718612 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A3AC4,,0x1F13,64,16,3979,52,53248,,,,
+01/11/17-20:49:38.718937 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A3AC4,0x56901436,,0x1000,64,10,43647,1500,225300,,,,
+01/11/17-20:49:38.718952 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A406C,0x56901436,,0x1000,64,10,31758,1500,225300,,,,
+01/11/17-20:49:38.718955 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A4614,,0x1F13,64,16,3980,52,53248,,,,
+01/11/17-20:49:38.718963 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4A,***AP***,0x1E3A4614,0x56901436,,0x1000,64,10,22512,60,61440,,,,
+01/11/17-20:49:38.719298 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A461C,0x56901436,,0x1000,64,10,21591,1500,225300,,,,
+01/11/17-20:49:38.719302 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A4BC4,,0x1F13,64,16,3981,52,53248,,,,
+01/11/17-20:49:38.719322 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E3A4BC4,0x56901436,,0x1000,64,10,59325,644,135176,,,,
+01/11/17-20:49:38.719683 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A4E14,0x56901436,,0x1000,64,10,60528,1500,225300,,,,
+01/11/17-20:49:38.719689 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A53BC,,0x1F13,64,16,3982,52,53248,,,,
+01/11/17-20:49:38.719707 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E3A53BC,0x56901436,,0x1000,64,10,51009,900,135180,,,,
+01/11/17-20:49:38.719709 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A570C,0x56901436,,0x1000,64,10,10473,92,94208,,,,
+01/11/17-20:49:38.719789 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A5734,,0x1F13,64,16,3983,52,53248,,,,
+01/11/17-20:49:38.720104 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A5734,0x5690144E,,0xFFF,64,8,62478,52,53248,,,,
+01/11/17-20:49:38.720549 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A5734,0x5690144E,,0x1000,64,10,24986,1500,225300,,,,
+01/11/17-20:49:38.720564 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E3A5CDC,0x5690144E,,0x1000,64,10,53439,292,36868,,,,
+01/11/17-20:49:38.720567 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A5DCC,,0x1F13,64,16,3985,52,53248,,,,
+01/11/17-20:49:38.720952 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A5DCC,0x5690144E,,0x1000,64,10,59946,1500,225300,,,,
+01/11/17-20:49:38.720973 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E3A6374,0x5690144E,,0x1000,64,10,27663,468,217092,,,,
+01/11/17-20:49:38.720977 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A6514,,0x1F13,64,16,3986,52,53248,,,,
+01/11/17-20:49:38.721423 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A6514,0x5690144E,,0x1000,64,10,6032,1500,225300,,,,
+01/11/17-20:49:38.721430 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E3A6ABC,0x5690144E,,0x1000,64,10,21413,164,167936,,,,
+01/11/17-20:49:38.721433 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A6B2C,,0x1F13,64,16,3987,52,53248,,,,
+01/11/17-20:49:38.721836 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A6B2C,0x5690144E,,0x1000,64,10,28700,1500,225300,,,,
+01/11/17-20:49:38.721841 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E3A70D4,0x5690144E,,0x1000,64,10,11267,788,20492,,,,
+01/11/17-20:49:38.721844 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A73B4,,0x1F13,64,16,3988,52,53248,,,,
+01/11/17-20:49:38.722366 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A73B4,0x5690144E,,0x1000,64,10,23915,1500,225300,,,,
+01/11/17-20:49:38.722373 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E3A795C,0x5690144E,,0x1000,64,10,22812,948,184332,,,,
+01/11/17-20:49:38.722377 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A7CDC,,0x1F13,64,16,3989,52,53248,,,,
+01/11/17-20:49:38.722666 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A7CDC,0x5690144E,,0x1000,64,10,56330,1500,225300,,,,
+01/11/17-20:49:38.722669 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E3A8284,0x5690144E,,0x1000,64,10,9418,964,200716,,,,
+01/11/17-20:49:38.722672 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8614,0x56901466,,0xFFF,64,8,54552,52,53248,,,,
+01/11/17-20:49:38.722709 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901466,0x1E3A8614,,0x1F13,64,16,3991,52,53248,,,,
+01/11/17-20:49:38.723188 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A8614,0x56901466,,0x1000,64,10,36476,1500,225300,,,,
+01/11/17-20:49:38.723194 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E3A8BBC,0x56901466,,0x1000,64,10,5160,756,249864,,,,
+01/11/17-20:49:38.723197 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901466,0x1E3A8E7C,,0x1F13,64,16,3992,52,53248,,,,
+01/11/17-20:49:38.723727 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8E7C,0x5690147E,,0xFFF,64,8,58139,52,53248,,,,
+01/11/17-20:49:38.723829 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A8E7C,0x5690147E,,0x1000,64,10,13602,92,94208,,,,
+01/11/17-20:49:38.724393 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8EA4,0x56901496,,0xFFF,64,8,7887,52,53248,,,,
+01/11/17-20:49:38.724427 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A8EA4,0x56901496,,0x1000,64,10,26953,76,77824,,,,
+01/11/17-20:49:38.725021 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8EBC,0x569014DE,,0xFFD,64,8,9083,52,53248,,,,
+01/11/17-20:49:38.725028 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A8EBC,0x569014DE,,0x1000,64,10,25776,76,77824,,,,
+01/11/17-20:49:38.739469 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A8ED4,0x569014DE,,0x1000,64,10,30246,92,94208,,,,
+01/11/17-20:49:38.739600 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569014DE,0x1E3A8EFC,,0x1F13,64,16,3996,52,53248,,,,
+01/11/17-20:49:38.739910 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8EFC,0x569014F6,,0xFFF,64,8,41919,52,53248,,,,
+01/11/17-20:49:38.740117 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E3A8EFC,0x569014F6,,0x1000,64,10,37312,556,45064,,,,
+01/11/17-20:49:38.741314 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x5690153E,,0xFFD,64,8,17721,52,53248,,,,
+01/11/17-20:49:38.771156 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x56901576,,0xFFE,64,8,55944,52,53248,,,,
+01/11/17-20:49:38.829875 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x569015BE,,0xFFD,64,8,4622,52,53248,,,,
+01/11/17-20:49:38.838484 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x5690161E,,0xFFD,64,8,12690,52,53248,,,,
+01/11/17-20:49:38.838557 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A90F4,0x5690161E,,0x1000,64,10,6352,76,77824,,,,
+01/11/17-20:49:38.864196 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A910C,0x5690161E,,0x1000,64,10,42677,76,77824,,,,
+01/11/17-20:49:38.864547 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690161E,0x1E3A9124,,0x1F13,64,16,4002,52,53248,,,,
+01/11/17-20:49:38.864823 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A9124,0x56901636,,0xFFF,64,8,62716,52,53248,,,,
+01/11/17-20:49:38.864973 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E3A9124,0x56901636,,0x1000,64,10,18619,204,208896,,,,
+01/11/17-20:49:38.866516 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91BC,0x5690167E,,0xFFD,64,8,13402,52,53248,,,,
+01/11/17-20:49:38.884253 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91BC,0x569016E6,,0xFFC,64,8,23445,52,53248,,,,
+01/11/17-20:49:38.884262 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91BC,0x56901716,,0xFFE,64,8,15998,52,53248,,,,
+01/11/17-20:49:38.884421 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A91BC,0x56901716,,0x1000,64,10,37876,76,77824,,,,
+01/11/17-20:49:38.907086 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A91D4,0x56901716,,0x1000,64,10,41998,92,94208,,,,
+01/11/17-20:49:38.907273 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901716,0x1E3A91FC,,0x1F13,64,16,4007,52,53248,,,,
+01/11/17-20:49:38.907498 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91FC,0x5690172E,,0xFFF,64,8,2031,52,53248,,,,
+01/11/17-20:49:38.907674 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E3A91FC,0x5690172E,,0x1000,64,10,65433,116,118784,,,,
+01/11/17-20:49:38.909097 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A923C,0x5690175E,,0xFFE,64,8,41174,52,53248,,,,
+01/11/17-20:49:38.909108 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A923C,0x5690175E,,0x1000,64,10,28977,92,94208,,,,
+01/11/17-20:49:38.950050 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690175E,0x1E3A9264,,0x1F13,64,16,4010,52,53248,,,,
+01/11/17-20:49:38.952099 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A9264,0x56901776,,0xFFF,64,8,20145,52,53248,,,,
+01/11/17-20:49:38.952254 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A9264,0x56901776,,0x1000,64,10,1604,92,94208,,,,
+01/11/17-20:49:38.952898 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901776,0x1E3A928C,,0x1F13,64,16,4012,52,53248,,,,
+01/11/17-20:49:38.955697 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A928C,0x5690178E,,0xFFF,64,8,29670,52,53248,,,,
+01/11/17-20:49:38.955918 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A928C,0x5690178E,,0x1000,64,10,41295,92,94208,,,,
+01/11/17-20:49:38.957143 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A92B4,0x569017A6,,0xFFF,64,8,53853,52,53248,,,,
+01/11/17-20:49:38.965865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A92B4,0x569017A6,,0x1000,64,10,34049,92,94208,,,,
+01/11/17-20:49:38.969753 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A92DC,0x569017BE,,0xFFF,64,8,18267,52,53248,,,,
+01/11/17-20:49:38.970498 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A92DC,0x569017BE,,0x1000,64,10,33394,1500,225300,,,,
+01/11/17-20:49:38.970508 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E3A9884,0x569017BE,,0x1000,64,10,24966,1060,36880,,,,
+01/11/17-20:49:38.970511 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3A9C74,,0x1F13,64,16,4016,52,53248,,,,
+01/11/17-20:49:38.971108 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A9C74,0x569017BE,,0x1000,64,10,2345,1500,225300,,,,
+01/11/17-20:49:38.971119 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AA21C,0x569017BE,,0x1000,64,10,63264,1500,225300,,,,
+01/11/17-20:49:38.971123 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AA7C4,,0x1F13,64,16,4017,52,53248,,,,
+01/11/17-20:49:38.971144 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AA7C4,0x569017BE,,0x1000,64,10,38086,76,77824,,,,
+01/11/17-20:49:38.971566 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AA7DC,0x569017BE,,0x1000,64,10,15052,1500,225300,,,,
+01/11/17-20:49:38.971575 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AAD84,,0x1F13,64,16,4018,52,53248,,,,
+01/11/17-20:49:38.971602 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E3AAD84,0x569017BE,,0x1000,64,10,23674,644,135176,,,,
+01/11/17-20:49:38.971871 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AAFD4,0x569017BE,,0x1000,64,10,48549,1500,225300,,,,
+01/11/17-20:49:38.971876 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AB57C,,0x1F13,64,16,4019,52,53248,,,,
+01/11/17-20:49:38.971888 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E3AB57C,0x569017BE,,0x1000,64,10,2232,900,135180,,,,
+01/11/17-20:49:38.971890 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3AB8CC,0x569017BE,,0x1000,64,10,11241,92,94208,,,,
+01/11/17-20:49:38.974036 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AB8F4,,0x1F13,64,16,4020,52,53248,,,,
+01/11/17-20:49:38.974671 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AB8F4,0x569017D6,,0xFFF,64,8,57358,52,53248,,,,
+01/11/17-20:49:38.974986 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AB8F4,0x569017D6,,0x1000,64,10,60275,1500,225300,,,,
+01/11/17-20:49:38.974992 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E3ABE9C,0x569017D6,,0x1000,64,10,29505,292,36868,,,,
+01/11/17-20:49:38.974995 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3ABF8C,,0x1F13,64,16,4022,52,53248,,,,
+01/11/17-20:49:38.975462 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3ABF8C,0x569017D6,,0x1000,64,10,62499,1500,225300,,,,
+01/11/17-20:49:38.975466 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E3AC534,0x569017D6,,0x1000,64,10,47337,468,217092,,,,
+01/11/17-20:49:38.975469 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3AC6D4,,0x1F13,64,16,4023,52,53248,,,,
+01/11/17-20:49:38.975865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AC6D4,0x569017D6,,0x1000,64,10,33384,1500,225300,,,,
+01/11/17-20:49:38.975870 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E3ACC7C,0x569017D6,,0x1000,64,10,34219,164,167936,,,,
+01/11/17-20:49:38.975873 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3ACCEC,,0x1F13,64,16,4024,52,53248,,,,
+01/11/17-20:49:38.976279 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3ACCEC,0x569017D6,,0x1000,64,10,39935,1500,225300,,,,
+01/11/17-20:49:38.976284 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E3AD294,0x569017D6,,0x1000,64,10,65405,788,20492,,,,
+01/11/17-20:49:38.976287 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3AD574,,0x1F13,64,16,4025,52,53248,,,,
+01/11/17-20:49:38.976672 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AD574,0x569017D6,,0x1000,64,10,47260,1500,225300,,,,
+01/11/17-20:49:38.976677 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E3ADB1C,0x569017D6,,0x1000,64,10,7038,948,184332,,,,
+01/11/17-20:49:38.976680 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3ADE9C,,0x1F13,64,16,4026,52,53248,,,,
+01/11/17-20:49:38.977244 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3ADE9C,0x569017D6,,0x1000,64,10,57825,1500,225300,,,,
+01/11/17-20:49:38.977252 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E3AE444,0x569017D6,,0x1000,64,10,14750,964,200716,,,,
+01/11/17-20:49:38.977256 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3AE7D4,,0x1F13,64,16,4027,52,53248,,,,
+01/11/17-20:49:38.977582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AE7D4,0x569017EE,,0xFFF,64,8,40965,52,53248,,,,
+01/11/17-20:49:38.977740 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AE7D4,0x569017EE,,0x1000,64,10,25235,1500,225300,,,,
+01/11/17-20:49:38.977746 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E3AED7C,0x569017EE,,0x1000,64,10,37899,756,249864,,,,
+01/11/17-20:49:38.977748 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017EE,0x1E3AF03C,,0x1F13,64,16,4029,52,53248,,,,
+01/11/17-20:49:38.978306 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF03C,0x56901806,,0xFFF,64,8,5277,52,53248,,,,
+01/11/17-20:49:38.978429 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3AF03C,0x56901806,,0x1000,64,10,39595,92,94208,,,,
+01/11/17-20:49:38.978660 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF064,0x5690181E,,0xFFF,64,8,18434,52,53248,,,,
+01/11/17-20:49:38.978743 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AF064,0x5690181E,,0x1000,64,10,59548,76,77824,,,,
+01/11/17-20:49:38.979085 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF07C,0x56901866,,0xFFD,64,8,64094,52,53248,,,,
+01/11/17-20:49:38.979195 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AF07C,0x56901866,,0x1000,64,10,27814,76,77824,,,,
+01/11/17-20:49:38.996191 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3AF094,0x56901866,,0x1000,64,10,58142,92,94208,,,,
+01/11/17-20:49:38.996344 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901866,0x1E3AF0BC,,0x1F13,64,16,4033,52,53248,,,,
+01/11/17-20:49:38.996582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF0BC,0x5690187E,,0xFFF,64,8,40934,52,53248,,,,
+01/11/17-20:49:38.996788 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E3AF0BC,0x5690187E,,0x1000,64,10,41880,556,45064,,,,
+01/11/17-20:49:38.998362 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x569018C6,,0xFFD,64,8,46597,52,53248,,,,
+01/11/17-20:49:39.029755 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x569018FE,,0xFFE,64,8,50704,52,53248,,,,
+01/11/17-20:49:39.090878 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x56901946,,0xFFD,64,8,58336,52,53248,,,,
+01/11/17-20:49:39.103926 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x569019A6,,0xFFD,64,8,6076,52,53248,,,,
+01/11/17-20:49:39.103935 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AF2B4,0x569019A6,,0x1000,64,10,7481,76,77824,,,,
+01/11/17-20:49:39.144528 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569019A6,0x1E3AF2CC,,0x1F13,64,16,4039,52,53248,,,,
+01/11/17-20:49:41.685007 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,12****S*,0x89DE3802,0x0,,0xFFFF,64,0,37837,64,65536,,,,
+01/11/17-20:49:41.685041 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50181,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,*2*A**S*,0x412FE1F9,0x89DE3803,,0x3890,64,0,0,60,61440,,,,
+01/11/17-20:49:41.685270 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3803,0x412FE1FA,,0x1015,64,0,45378,52,53248,,,,
+01/11/17-20:49:41.685291 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x203,***AP***,0x89DE3803,0x412FE1FA,,0x1015,64,2,31618,501,250884,,,,
+01/11/17-20:49:41.685297 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50181,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x412FE1FA,0x89DE39C4,,0x7A,64,0,35921,52,53248,,,,
+01/11/17-20:49:41.702785 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE39C4,0x412FEAB5,,0xFE7,64,0,31331,52,53248,,,,
+01/11/17-20:49:41.708304 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E0,***AP***,0x89DE39C4,0x412FEAB5,,0x1000,64,2,55934,466,215044,,,,
+01/11/17-20:49:41.708865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,12****S*,0xE53A56EE,0x0,,0xFFFF,64,0,35683,64,65536,,,,
+01/11/17-20:49:41.708880 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50182,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,*2*A**S*,0x18588B4A,0xE53A56EF,,0x3890,64,0,0,60,61440,,,,
+01/11/17-20:49:41.709029 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A56EF,0x18588B4B,,0x1015,64,0,32055,52,53248,,,,
+01/11/17-20:49:41.709146 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1DD,***AP***,0xE53A56EF,0x18588B4B,,0x1015,64,2,10852,463,211972,,,,
+01/11/17-20:49:41.709154 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50182,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x18588B4B,0xE53A588A,,0x7A,64,0,59016,52,53248,,,,
+01/11/17-20:49:41.710292 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,12****S*,0x8365AF51,0x0,,0xFFFF,64,0,7652,64,65536,,,,
+01/11/17-20:49:41.710309 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50183,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,*2*A**S*,0x8DE9822F,0x8365AF52,,0x3890,64,0,0,60,61440,,,,
+01/11/17-20:49:41.710470 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365AF52,0x8DE98230,,0x1015,64,0,60499,52,53248,,,,
+01/11/17-20:49:41.710664 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1D0,***AP***,0x8365AF52,0x8DE98230,,0x1015,64,2,49526,450,198660,,,,
+01/11/17-20:49:41.710674 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50183,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x8DE98230,0x8365B0E0,,0x7A,64,0,61429,52,53248,,,,
+01/11/17-20:49:41.713117 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,******S*,0x39818AB4,0x0,,0xFFFF,64,0,31071,64,65536,,,,
+01/11/17-20:49:41.713131 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50184,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,***A**S*,0xA874A3E9,0x39818AB5,,0x3890,64,0,0,60,61440,,,,
+01/11/17-20:49:41.713287 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818AB5,0xA874A3EA,,0x1015,64,0,63436,52,53248,,,,
+01/11/17-20:49:41.713427 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1CD,***AP***,0x39818AB5,0xA874A3EA,,0x1015,64,0,50310,447,195588,,,,
+01/11/17-20:49:41.713435 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50184,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0xA874A3EA,0x39818C40,,0x7A,64,0,52781,52,53248,,,,
+01/11/17-20:49:41.735024 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE98D80,,0xFBB,64,0,60492,52,53248,,,,
+01/11/17-20:49:41.735121 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE998D0,,0xFA5,64,0,27946,52,53248,,,,
+01/11/17-20:49:41.735230 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9A420,,0xF4B,64,0,12533,52,53248,,,,
+01/11/17-20:49:41.735368 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9AF70,,0xEF0,64,0,61911,52,53248,,,,
+01/11/17-20:49:41.735487 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9BAC0,,0xE96,64,0,32463,52,53248,,,,
+01/11/17-20:49:41.735764 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9C610,,0xE3B,64,0,41222,52,53248,,,,
+01/11/17-20:49:41.735771 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9C610,,0x1000,64,0,17505,52,53248,,,,
+01/11/17-20:49:41.735934 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9D160,,0xFA5,64,0,6378,52,53248,,,,
+01/11/17-20:49:41.735939 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9DCB0,,0xF4B,64,0,50978,52,53248,,,,
+01/11/17-20:49:41.736031 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9E800,,0xEF0,64,0,41992,52,53248,,,,
+01/11/17-20:49:41.736034 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9F350,,0xE96,64,0,40068,52,53248,,,,
+01/11/17-20:49:41.736039 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9FEA0,,0xE3B,64,0,4465,52,53248,,,,
+01/11/17-20:49:41.736130 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA0365,,0xE15,64,0,64175,52,53248,,,,
+01/11/17-20:49:41.736242 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA0365,,0x1000,64,0,60532,52,53248,,,,
+01/11/17-20:49:41.736790 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA0798,,0xFDE,64,0,1924,52,53248,,,,
+01/11/17-20:49:41.740496 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA079E,,0xFFF,64,0,12835,52,53248,,,,
+01/11/17-20:49:41.747598 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50181,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x412FEAB5,0x89DE3B62,,0x82,64,0,35924,52,53248,,,,
+01/11/17-20:49:41.757549 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858969B,,0xFBB,64,0,20679,52,53248,,,,
+01/11/17-20:49:41.758290 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858A1EB,,0xFA5,64,0,44022,52,53248,,,,
+01/11/17-20:49:41.758399 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858AD3B,,0xF4B,64,0,28875,52,53248,,,,
+01/11/17-20:49:41.758404 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858B88B,,0xEF0,64,0,43317,52,53248,,,,
+01/11/17-20:49:41.758405 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858C3DB,,0xE96,64,0,55975,52,53248,,,,
+01/11/17-20:49:41.758406 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858C3DB,,0x1000,64,0,19782,52,53248,,,,
+01/11/17-20:49:41.758408 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858CF2B,,0xFA5,64,0,63139,52,53248,,,,
+01/11/17-20:49:41.759044 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858CF2B,,0x1000,64,0,49835,52,53248,,,,
+01/11/17-20:49:41.771340 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x412FF605,,0xFA5,64,0,44665,52,53248,,,,
+01/11/17-20:49:41.773636 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41300155,,0xF4B,64,0,6905,52,53248,,,,
+01/11/17-20:49:41.773648 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41300CA5,,0xEF0,64,0,61971,52,53248,,,,
+01/11/17-20:49:41.773653 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x413017F5,,0xE96,64,0,31949,52,53248,,,,
+01/11/17-20:49:41.773656 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302345,,0xE3B,64,0,48816,52,53248,,,,
+01/11/17-20:49:41.773658 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302345,,0xEBB,64,0,30870,52,53248,,,,
+01/11/17-20:49:41.773660 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302345,,0x1000,64,0,8585,52,53248,,,,
+01/11/17-20:49:41.773662 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302E95,,0xFA5,64,0,61708,52,53248,,,,
+01/11/17-20:49:41.773664 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302E95,,0x1000,64,0,38538,52,53248,,,,
+01/11/17-20:49:41.777099 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858DA7B,,0xFA5,64,0,23510,52,53248,,,,
+01/11/17-20:49:41.777115 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858E5CB,,0xF4B,64,0,16857,52,53248,,,,
+01/11/17-20:49:41.777119 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858F11B,,0xEF0,64,0,34551,52,53248,,,,
+01/11/17-20:49:41.777122 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858FC6B,,0xE96,64,0,63082,52,53248,,,,
+01/11/17-20:49:41.777127 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185907BB,,0xE3B,64,0,37755,52,53248,,,,
+01/11/17-20:49:41.777129 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590C70,,0xE15,64,0,42363,52,53248,,,,
+01/11/17-20:49:41.777132 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590F7F,,0xDFD,64,0,8363,52,53248,,,,
+01/11/17-20:49:41.777134 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x413039E5,,0xFA5,64,0,11423,52,53248,,,,
+01/11/17-20:49:41.777248 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41304535,,0xF4B,64,0,53395,52,53248,,,,
+01/11/17-20:49:41.777254 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41305085,,0xEF0,64,0,42374,52,53248,,,,
+01/11/17-20:49:41.777256 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41305BD5,,0xE96,64,0,27546,52,53248,,,,
+01/11/17-20:49:41.777259 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41306725,,0xE3B,64,0,42745,52,53248,,,,
+01/11/17-20:49:41.777261 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41306BDA,,0xE15,64,0,64435,52,53248,,,,
+01/11/17-20:49:41.777264 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590F7F,,0x1000,64,0,43557,52,53248,,,,
+01/11/17-20:49:41.777266 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41306BDA,,0x1000,64,0,46958,52,53248,,,,
+01/11/17-20:49:41.778744 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x4130772A,,0xFA5,64,0,14812,52,53248,,,,
+01/11/17-20:49:41.778811 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41307BED,,0xF7F,64,0,16449,52,53248,,,,
+01/11/17-20:49:41.778817 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41307BED,,0x1000,64,0,32765,52,53248,,,,
+01/11/17-20:49:41.779807 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41307BF3,,0xFFF,64,0,61783,52,53248,,,,
+01/11/17-20:49:41.779816 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590F85,,0xFFF,64,0,21799,52,53248,,,,
+01/11/17-20:49:41.780562 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874AF3A,,0xFBB,64,0,41668,52,53248,,,,
+01/11/17-20:49:41.780593 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874BA8A,,0xF60,64,0,44575,52,53248,,,,
+01/11/17-20:49:41.780611 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874BA8A,,0xFCB,64,0,60316,52,53248,,,,
+01/11/17-20:49:41.780614 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874C5DA,,0xF70,64,0,29140,52,53248,,,,
+01/11/17-20:49:41.781366 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874D12A,,0xF16,64,0,29289,52,53248,,,,
+01/11/17-20:49:41.781471 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874DC7A,,0xEBB,64,0,29562,52,53248,,,,
+01/11/17-20:49:41.781478 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874E7CA,,0xE61,64,0,48236,52,53248,,,,
+01/11/17-20:49:41.781624 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874F31A,,0xE06,64,0,39280,52,53248,,,,
+01/11/17-20:49:41.781630 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874FE6A,,0xDAC,64,0,8700,52,53248,,,,
+01/11/17-20:49:41.781633 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874FE6A,,0x1000,64,0,10159,52,53248,,,,
+01/11/17-20:49:41.781924 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA87509BA,,0xFA5,64,0,11390,52,53248,,,,
+01/11/17-20:49:41.781933 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA87509BA,,0x1000,64,0,24685,52,53248,,,,
+01/11/17-20:49:41.781936 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875150A,,0xFA5,64,0,3366,52,53248,,,,
+01/11/17-20:49:41.781938 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875205A,,0xF4B,64,0,36802,52,53248,,,,
+01/11/17-20:49:41.781941 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875205A,,0x1000,64,0,13880,52,53248,,,,
+01/11/17-20:49:41.781943 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8752BAA,,0xFA5,64,0,27392,52,53248,,,,
+01/11/17-20:49:41.781945 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875360B,,0xF52,64,0,28642,52,53248,,,,
+01/11/17-20:49:41.781949 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8753611,,0xF52,64,0,55533,52,53248,,,,
+01/11/17-20:49:41.782121 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8753611,,0x1000,64,0,43032,52,53248,,,,
+01/11/17-20:49:41.799072 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA12EE,,0xFA5,64,0,30116,52,53248,,,,
+01/11/17-20:49:41.799086 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA1E3E,,0xF4B,64,0,57836,52,53248,,,,
+01/11/17-20:49:41.799089 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41308743,,0xFA5,64,0,45743,52,53248,,,,
+01/11/17-20:49:41.799096 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309293,,0xF4B,64,0,19041,52,53248,,,,
+01/11/17-20:49:41.799099 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309BF3,,0xF00,64,0,56110,52,53248,,,,
+01/11/17-20:49:41.799101 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309F20,,0xEE6,64,0,14796,52,53248,,,,
+01/11/17-20:49:41.799103 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA279E,,0xF00,64,0,42649,52,53248,,,,
+01/11/17-20:49:41.799106 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA279E,,0x1000,64,0,13788,52,53248,,,,
+01/11/17-20:49:41.799107 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309F20,,0x1000,64,0,44776,52,53248,,,,
+01/11/17-20:49:41.799109 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309F25,,0xFFF,64,0,2086,52,53248,,,,
+01/11/17-20:49:41.799832 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18591AD5,,0xFA5,64,0,64984,52,53248,,,,
+01/11/17-20:49:41.799898 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18591AD5,,0x1000,64,0,3933,52,53248,,,,
+01/11/17-20:49:41.799900 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592625,,0xFA5,64,0,16728,52,53248,,,,
+01/11/17-20:49:41.800244 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592F85,,0xF5A,64,0,16089,52,53248,,,,
+01/11/17-20:49:41.800252 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592F85,,0x1000,64,0,49727,52,53248,,,,
+01/11/17-20:49:41.800410 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592F8D,,0xFFF,64,0,46722,52,53248,,,,
+01/11/17-20:49:41.800418 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA27A6,,0xFFF,64,0,44196,52,53248,,,,
+01/11/17-20:49:41.800513 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18593ADD,,0xFA5,64,0,62171,52,53248,,,,
+01/11/17-20:49:41.800519 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185944C2,,0xF56,64,0,16322,52,53248,,,,
+01/11/17-20:49:41.800523 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185944C9,,0xF55,64,0,48908,52,53248,,,,
+01/11/17-20:49:41.800525 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA32F6,,0xFA5,64,0,41536,52,53248,,,,
+01/11/17-20:49:41.800559 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA3E46,,0xF4A,64,0,5801,52,53248,,,,
+01/11/17-20:49:41.800563 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185944C9,,0x1000,64,0,38774,52,53248,,,,
+01/11/17-20:49:41.800565 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA47A6,,0xEFF,64,0,15422,52,53248,,,,
+01/11/17-20:49:41.800567 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA47A6,,0x1000,64,0,5924,52,53248,,,,
+01/11/17-20:49:41.809759 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA47AD,,0xFFF,64,0,44728,52,53248,,,,
+01/11/17-20:49:41.809787 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA5293,,0xFA8,64,0,19718,52,53248,,,,
+01/11/17-20:49:41.810740 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8754161,,0xFA5,64,0,7993,52,53248,,,,
+01/11/17-20:49:41.810762 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8754161,,0x1000,64,0,58879,52,53248,,,,
+01/11/17-20:49:41.810764 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8754CB1,,0xFA5,64,0,785,52,53248,,,,
+01/11/17-20:49:41.810767 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8755611,,0xF5A,64,0,54091,52,53248,,,,
+01/11/17-20:49:41.812571 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8755611,,0x1000,64,0,27089,52,53248,,,,
+01/11/17-20:49:41.812579 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8755619,,0xFFF,64,0,50244,52,53248,,,,
+01/11/17-20:49:41.812584 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8756169,,0xFA5,64,0,6997,52,53248,,,,
+01/11/17-20:49:41.812586 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8756169,,0x1000,64,0,45003,52,53248,,,,
+01/11/17-20:49:41.812587 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8756CB9,,0xFA5,64,0,18800,52
<TRUNCATED>
[46/50] [abbrv] metron git commit: METRON-1455: Patch and Replace
methods in the REST UpdateController return 400 this closes
apache/incubator-metron#937
Posted by rm...@apache.org.
METRON-1455: Patch and Replace methods in the REST UpdateController return 400 this closes apache/incubator-metron#937
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e59059bd
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e59059bd
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e59059bd
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: e59059bd9707a6ca46c4137d796b8f2943f06b43
Parents: 15b98de
Author: cstella <ce...@gmail.com>
Authored: Thu Feb 15 13:00:36 2018 -0500
Committer: cstella <ce...@gmail.com>
Committed: Thu Feb 15 13:00:36 2018 -0500
----------------------------------------------------------------------
.../apache/metron/common/utils/JSONUtils.java | 7 ++++++
.../dao/ElasticsearchMetaAlertDao.java | 15 ++++++++-----
.../apache/metron/indexing/dao/IndexDao.java | 23 +++++---------------
.../indexing/dao/update/PatchRequest.java | 7 +++---
4 files changed, 26 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
index 135546d..c02f19d 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/JSONUtils.java
@@ -184,4 +184,11 @@ public enum JSONUtils {
return toJSONPretty(JsonPatch.apply(patchNode, sourceNode));
}
+ public Map<String, Object> applyPatch(List<Map<String, Object>> patch, Map<String, Object> source) {
+ JsonNode originalNode = convert(source, JsonNode.class);
+ JsonNode patchNode = convert(patch, JsonNode.class);
+ JsonNode patched = JsonPatch.apply(patchNode, originalNode);
+ return _mapper.get().convertValue(patched, new TypeReference<Map<String, Object>>() { });
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
index 9740272..2311a2b 100644
--- a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
+++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/dao/ElasticsearchMetaAlertDao.java
@@ -473,12 +473,15 @@ public class ElasticsearchMetaAlertDao implements MetaAlertDao {
}
protected boolean isPatchAllowed(PatchRequest request) {
- Iterator patchIterator = request.getPatch().iterator();
- while(patchIterator.hasNext()) {
- JsonNode patch = (JsonNode) patchIterator.next();
- String path = patch.path("path").asText();
- if (STATUS_PATH.equals(path) || ALERT_PATH.equals(path)) {
- return false;
+ if(request.getPatch() != null && !request.getPatch().isEmpty()) {
+ for(Map<String, Object> patch : request.getPatch()) {
+ Object pathObj = patch.get("path");
+ if(pathObj != null && pathObj instanceof String) {
+ String path = (String)pathObj;
+ if (STATUS_PATH.equals(path) || ALERT_PATH.equals(path)) {
+ return false;
+ }
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
index 2c057d4..fe546bd 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/IndexDao.java
@@ -17,16 +17,11 @@
*/
package org.apache.metron.indexing.dao;
-import com.fasterxml.jackson.annotation.JsonInclude;
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.JsonNode;
import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.flipkart.zjsonpatch.JsonPatch;
import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.indexing.dao.search.FieldType;
import org.apache.metron.indexing.dao.search.GetRequest;
@@ -46,9 +41,6 @@ import org.apache.metron.indexing.dao.update.ReplaceRequest;
*/
public interface IndexDao {
- public static ThreadLocal<ObjectMapper> _mapper = ThreadLocal.withInitial(() ->
- new ObjectMapper().setSerializationInclusion(JsonInclude.Include.NON_NULL));
-
/**
* Return search response based on the search request
*
@@ -136,7 +128,7 @@ public interface IndexDao {
default Document getPatchedDocument(PatchRequest request
, Optional<Long> timestamp
- ) throws OriginalNotFoundException, IOException {
+ ) throws OriginalNotFoundException, IOException {
Map<String, Object> latest = request.getSource();
if(latest == null) {
Document latestDoc = getLatest(request.getGuid(), request.getSensorType());
@@ -147,14 +139,11 @@ public interface IndexDao {
throw new OriginalNotFoundException("Unable to patch an document that doesn't exist and isn't specified.");
}
}
- JsonNode originalNode = _mapper.get().convertValue(latest, JsonNode.class);
- JsonNode patched = JsonPatch.apply(request.getPatch(), originalNode);
- Map<String, Object> updated = _mapper.get()
- .convertValue(patched, new TypeReference<Map<String, Object>>() {});
- return new Document( updated
- , request.getGuid()
- , request.getSensorType()
- , timestamp.orElse(System.currentTimeMillis()));
+ Map<String, Object> updated = JSONUtils.INSTANCE.applyPatch(request.getPatch(), latest);
+ return new Document(updated
+ , request.getGuid()
+ , request.getSensorType()
+ , timestamp.orElse(System.currentTimeMillis()));
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/e59059bd/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
index 77f5958..6650666 100644
--- a/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
+++ b/metron-platform/metron-indexing/src/main/java/org/apache/metron/indexing/dao/update/PatchRequest.java
@@ -19,10 +19,11 @@ package org.apache.metron.indexing.dao.update;
import com.fasterxml.jackson.databind.JsonNode;
+import java.util.List;
import java.util.Map;
public class PatchRequest {
- JsonNode patch;
+ List<Map<String, Object>> patch;
Map<String, Object> source;
String guid;
String sensorType;
@@ -54,11 +55,11 @@ public class PatchRequest {
* </pre>
* @return
*/
- public JsonNode getPatch() {
+ public List<Map<String, Object>> getPatch() {
return patch;
}
- public void setPatch(JsonNode patch) {
+ public void setPatch(List<Map<String, Object>> patch) {
this.patch = patch;
}
[50/50] [abbrv] metron git commit: Merge remote-tracking branch
'upstream/feature/METRON-1344-test-infrastructure' into
feature/METRON-1344-test-infrastructure
Posted by rm...@apache.org.
Merge remote-tracking branch 'upstream/feature/METRON-1344-test-infrastructure' into feature/METRON-1344-test-infrastructure
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/fc4ce010
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/fc4ce010
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/fc4ce010
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: fc4ce01047345b065214089b466babbdc1d5d30d
Parents: e265b36 ebc3258
Author: merrimanr <me...@gmail.com>
Authored: Thu Feb 22 14:12:16 2018 -0600
Committer: merrimanr <me...@gmail.com>
Committed: Thu Feb 22 14:12:16 2018 -0600
----------------------------------------------------------------------
.travis.yml | 19 +-
metron-contrib/metron-docker-e2e/.gitignore | 3 +
.../compose/docker-compose.yml | 55 +++++
.../compose/metron-centos/Dockerfile | 19 ++
.../compose/metron-rest/Dockerfile | 34 +++
.../compose/metron-rest/bin/start.sh | 23 ++
.../metron-rest/config/application-docker.yml | 62 ++++++
.../metron-rest/config/zookeeper/global.json | 8 +
.../compose/metron-ui/Dockerfile | 34 +++
.../compose/metron-ui/bin/start.sh | 20 ++
.../compose/metron-ui/config/alerts_ui.yml | 21 ++
metron-contrib/metron-docker-e2e/conf/.env | 3 +
metron-contrib/metron-docker-e2e/pom.xml | 217 +++++++++++++++++++
.../metron-docker-e2e/scripts/wait_for_rest.sh | 29 +++
metron-contrib/pom.xml | 1 +
.../configure-table/configure-table.e2e-spec.ts | 25 +--
.../metron-alerts/e2e/login/login.e2e-spec.ts | 3 +-
.../metron-alerts/e2e/login/login.po.ts | 2 +-
.../metron-alerts/e2e/utils/e2e_util.ts | 29 ++-
metron-interface/metron-alerts/pom.xml | 46 ++++
.../metron-alerts/protractor.conf.js | 21 +-
.../ElasticsearchSearchIntegrationTest.java | 86 +++++---
.../indexing/dao/SearchIntegrationTest.java | 90 +++++---
23 files changed, 742 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/.travis.yml
----------------------------------------------------------------------
diff --cc .travis.yml
index 71d7165,5bf4587..073dc1e
--- a/.travis.yml
+++ b/.travis.yml
@@@ -34,9 -47,19 +47,12 @@@ before_install
install:
- time mvn -q -T 2C -DskipTests clean install
+ - cd $E2E_COMPOSE_HOME && docker-compose up -d
+ - if [ ! -f ${DOCKER_METRON_CENTOS} ]; then docker save metron-centos | gzip > ${DOCKER_METRON_CENTOS}; fi
+ - cd $TRAVIS_BUILD_DIR
script:
- - ls -la
- - metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh localhost 8082
- - cd $E2E_COMPOSE_HOME && docker-compose logs metron-rest && cd ../../../
-# - mvn -q -T 2C surefire:test@unit-tests
-# - mvn -q surefire:test@integration-tests
-# - mvn -q test --projects metron-interface/metron-config
- - mvn test -Pe2e --projects=metron-interface/metron-alerts
-# - build_utils/verify_licenses.sh
+ - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time dev-utilities/build-utils/verify_licenses.sh
before_cache:
- rm -rf $HOME/.m2/repository/org/apache/metron
http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-contrib/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-interface/metron-alerts/e2e/utils/e2e_util.ts
----------------------------------------------------------------------
diff --cc metron-interface/metron-alerts/e2e/utils/e2e_util.ts
index 8ae1de1,7127daf..9304a5c
--- a/metron-interface/metron-alerts/e2e/utils/e2e_util.ts
+++ b/metron-interface/metron-alerts/e2e/utils/e2e_util.ts
@@@ -46,24 -46,36 +46,45 @@@ export function waitForStalenessOf (_el
}
export function loadTestData() {
- deleteTestData();
+ request.delete('http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', function (e, response, body) {
+ request.post({url:'http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', json:
+ {
+ "hdfs": {
+ "index": "alerts_ui_e2e",
+ "batchSize": 5,
+ "enabled": true
+ },
+ "elasticsearch": {
+ "index": "alerts_ui_e2e",
+ "batchSize": 5,
+ "enabled": true
+ },
+ "solr": {
+ "index": "alerts_ui_e2e",
+ "batchSize": 5,
+ "enabled": true
+ }
+ }
+ }, function (e, response, body) {
+ });
+ });
- request.delete('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index*', function (e, response, body) {
- fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.template')
- .pipe(request.post('http://' + browser.params.elasticsearch.url + '/_template/alerts_ui_e2e_index', function (e, response, body) {
- fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.data')
- .pipe(request.post('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk', function (e, response, body) {
- }));
- }));
+ let template = fs.readFileSync('e2e/mock-data/alerts_ui_e2e_index.template', 'utf8');
+ request({
+ url: 'http://node1:9200/_template/alerts_ui_e2e_index',
+ method: 'POST',
+ body: template
+ }, function(error, response, body) {
+ // add logging if desired
+ });
+
+ let data = fs.readFileSync('e2e/mock-data/alerts_ui_e2e_index.data', 'utf8');
+ request({
+ url: 'http://node1:9200/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk',
+ method: 'POST',
+ body: data
+ }, function(error, response, body) {
+ // add logging if desired
});
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-interface/metron-alerts/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
index 1bc5b6e,ae8dbe0..7089033
--- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
+++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java
@@@ -17,48 -17,47 +17,59 @@@
*/
package org.apache.metron.elasticsearch.integration;
-
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+ import java.util.Map;
import org.adrianwalker.multilinestring.Multiline;
+import org.apache.metron.common.utils.JSONUtils;
import org.apache.metron.elasticsearch.dao.ElasticsearchDao;
- import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent;
+ import org.apache.metron.elasticsearch.utils.ElasticsearchUtils;
import org.apache.metron.indexing.dao.AccessConfig;
import org.apache.metron.indexing.dao.IndexDao;
import org.apache.metron.indexing.dao.SearchIntegrationTest;
+import org.apache.metron.indexing.dao.search.GetRequest;
+import org.apache.metron.integration.InMemoryComponent;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
-import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexRequestBuilder;
+import org.elasticsearch.action.support.WriteRequest;
+ import org.elasticsearch.action.search.SearchResponse;
+ import org.elasticsearch.client.Client;
+ import org.elasticsearch.index.IndexNotFoundException;
+ import org.elasticsearch.index.query.QueryBuilders;
+ import org.elasticsearch.search.SearchHit;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
+import org.junit.Test;
- public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
+ import java.io.IOException;
+ import java.util.HashMap;
+ import java.util.concurrent.ExecutionException;
+ import org.junit.AfterClass;
+ import org.junit.BeforeClass;
- private static String indexDir = "target/elasticsearch_search";
+ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest {
+ private static String host = "localhost";
+ private static String port = "9310";
private static String dateFormat = "yyyy.MM.dd.HH";
private static final int MAX_RETRIES = 10;
private static final int SLEEP_MS = 500;
/**
* {
- * "bro_doc": {
+ * "searchintegrationtest_bro_doc": {
* "properties": {
* "source:type": {
- * "type": "string",
- * "index": "not_analyzed"
+ * "type": "text",
+ * "fielddata" : "true"
+ * },
+ * "guid" : {
+ * "type" : "keyword"
* },
* "ip_src_addr": {
* "type": "ip"
@@@ -105,14 -102,11 +116,14 @@@
/**
* {
- * "snort_doc": {
+ * "searchintegrationtest_snort_doc": {
* "properties": {
* "source:type": {
- * "type": "string",
- * "index": "not_analyzed"
+ * "type": "text",
+ * "fielddata" : "true"
+ * },
+ * "guid" : {
+ * "type" : "keyword"
* },
* "ip_src_addr": {
* "type": "ip"
@@@ -173,8 -163,34 +184,34 @@@
* }
*/
@Multiline
- private static String metaAlertTypeMappings;
+ private static String broDefaultStringMappings;
+ private static Map<String, Object> globalConfig;
+ private static Client client;
+
+ @BeforeClass
+ public static void start() {
+ globalConfig = new HashMap<String, Object>() {{
+ put("es.clustername", "elasticsearch");
+ put("es.port", port);
+ put("es.ip", host);
+ put("es.date.format", dateFormat);
+ }};
+ client = ElasticsearchUtils.getClient(globalConfig, null);
+ clearIndices();
+ }
+
+ @AfterClass
+ public static void stop() throws Exception {
+ clearIndices();
+ }
+
+ private static void clearIndices() {
+ try {
+ client.admin().indices().prepareDelete(broIndex, snortIndex, metaAlertIndex).get();
+ } catch (IndexNotFoundException infe) {}
+ }
+
@Override
protected IndexDao createDao() throws Exception {
AccessConfig config = new AccessConfig();
@@@ -195,25 -204,16 +225,14 @@@
}
@Override
- protected InMemoryComponent startIndex() throws Exception {
- InMemoryComponent es = new ElasticSearchComponent.Builder()
- .withHttpPort(9211)
- .withIndexDir(new File(indexDir))
- .build();
- es.start();
- return es;
- }
-
- @Override
protected void loadTestData()
throws ParseException, IOException, ExecutionException, InterruptedException {
- ElasticSearchComponent es = (ElasticSearchComponent)indexComponent;
- es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01")
- .addMapping("bro_doc", broTypeMappings).addMapping("bro_doc_default", broDefaultStringMappings).get();
- es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02")
- .addMapping("snort_doc", snortTypeMappings).get();
+ client.admin().indices().prepareCreate(broIndex)
+ .addMapping(broType, broTypeMappings).get();
+ client.admin().indices().prepareCreate(snortIndex)
+ .addMapping(snortType, snortTypeMappings).get();
- client.admin().indices().prepareCreate(metaAlertIndex)
- .addMapping(metaAlertType, metaAlertTypeMappings).get();
- BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
+ BulkRequestBuilder bulkRequest = client.prepareBulk().setRefresh(true);
JSONArray broArray = (JSONArray) new JSONParser().parse(broData);
for(Object o: broArray) {
JSONObject jsonObject = (JSONObject) o;
@@@ -236,7 -241,52 +253,6 @@@
if (bulkResponse.hasFailures()) {
throw new RuntimeException("Failed to index test data");
}
-
- SearchResponse broDocs = client
- .prepareSearch(broIndex)
- .setTypes(broType)
- .setQuery(QueryBuilders.matchAllQuery())
- .get();
- // We're changing the _id field, we need to create a copy and delete the original.
- for (SearchHit hit : broDocs.getHits()) {
- // Bro GUIDs to collide while using the standard analyzer
- // Use timestamp as part of guid because query may not return in order each time
- IndexRequest indexRequest = new IndexRequest()
- .index(broIndex)
- .type(broType)
- .id("bro-" + hit.getSource().get("timestamp"))
- .source(hit.getSource());
- client.index(indexRequest).get();
-
- // Delete the original
- client.prepareDelete(broIndex, broType, hit.getId())
- .get();
- }
-
- // Wait until everything is updated
- // Assume true until proven otherwise.
- boolean allUpdated = true;
- for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) {
- allUpdated = true;
- SearchResponse response = client
- .prepareSearch(broIndex)
- .setTypes(broType)
- .setQuery(QueryBuilders.matchAllQuery())
- .get();
- if (response.getHits().getTotalHits() == 0) {
- throw new IllegalStateException("Bro index is empty. No docs to validate were updated");
- }
- for (SearchHit hit : response.getHits()) {
- if (!hit.getId().startsWith("bro-")) {
- allUpdated = false;
- }
- }
- if (allUpdated) {
- break;
- }
- }
- if (!allUpdated) {
- throw new IllegalStateException("Unable to update Elasticsearch ids properly");
- }
}
+
-
}
http://git-wip-us.apache.org/repos/asf/metron/blob/fc4ce010/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
----------------------------------------------------------------------
diff --cc metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
index b40db46,84f6f40..d7aa7c7
--- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
+++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java
@@@ -41,11 -34,26 +41,20 @@@ import org.apache.metron.integration.In
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
+import org.junit.rules.ExpectedException;
public abstract class SearchIntegrationTest {
+
+ private static final String namespace = SearchIntegrationTest.class.getSimpleName().toLowerCase();
+ protected static final String broIndex = namespace + "_bro_index";
+ protected static final String snortIndex = namespace + "_snort_index";
+ protected static final String metaAlertIndex = namespace + "_metaalert_index";
+ protected static final String broType = namespace + "_bro_doc";
+ protected static final String snortType = namespace + "_snort_doc";
+ protected static final String metaAlertType = namespace + "_metaalert_doc";
+
/**
* [
* {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"},
@@@ -71,8 -79,17 +80,8 @@@
public static String snortData;
/**
- * [
- *{"guid":"meta_1","alert":[{"guid":"bro_1"}],"average":"5.0","min":"5.0","median":"5.0","max":"5.0","count":"1.0","sum":"5.0"},
- *{"guid":"meta_2","alert":[{"guid":"bro_1"},{"guid":"bro_2"},{"guid":"snort_1"}],"average":"5.0","min":"0.0","median":"5.0","max":"10.0","count":"3.0","sum":"15.0"}
- * ]
- */
- @Multiline
- public static String metaAlertData;
-
- /**
* {
- * "indices": ["bro", "snort"],
+ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"],
* "query": "*",
* "from": 0,
* "size": 10,
@@@ -89,8 -106,8 +98,8 @@@
/**
* {
- * "guid": "bro-3",
+ * "guid": "bro_3",
- * "sensorType": "bro"
+ * "sensorType": "searchintegrationtest_bro"
* }
*/
@Multiline
@@@ -99,12 -116,12 +108,12 @@@
/**
* [
* {
- * "guid": "bro-1",
+ * "guid": "bro_1",
- * "sensorType": "bro"
+ * "sensorType": "searchintegrationtest_bro"
* },
* {
- * "guid": "bro-2",
+ * "guid": "snort_2",
- * "sensorType": "snort"
+ * "sensorType": "searchintegrationtest_bro"
* }
* ]
*/
@@@ -222,7 -239,7 +231,7 @@@
/**
* {
* "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"],
- * "indices": ["bro", "snort"],
- * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"],
++ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"],
* "query": "*",
* "from": 0,
* "size": 10,
@@@ -328,7 -345,7 +337,7 @@@
/**
* {
* "fields": ["guid"],
- * "indices": ["bro"],
- * "indices": ["searchintegrationtest_metaalert"],
++ * "indices": ["searchintegrationtest_bro"],
* "query": "*",
* "from": 0,
* "size": 10,
@@@ -355,7 -372,7 +364,7 @@@
* }
* ],
* "scoreField":"score",
- * "indices": ["bro", "snort"],
- * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"],
++ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"],
* "query": "*"
* }
*/
@@@ -380,7 -397,7 +389,7 @@@
* }
* }
* ],
- * "indices": ["bro", "snort"],
- * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"],
++ * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"],
* "query": "*"
* }
*/
@@@ -431,269 -447,283 +439,291 @@@
}
}
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
@Test
- public void test() throws Exception {
- //All Query Testcase
- {
- SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, results.size());
- for(int i = 0;i < 5;++i) {
- Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
- Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp"));
- }
- for(int i = 5;i < 10;++i) {
- Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
- Assert.assertEquals(10-i, results.get(i).getSource().get("timestamp"));
- }
- }
- //Find One Guid Testcase
- {
- GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
- Optional<Map<String, Object>> response = dao.getLatestResult(request);
- Assert.assertTrue(response.isPresent());
- Map<String, Object> doc = response.get();
- Assert.assertEquals("bro", doc.get("source:type"));
- Assert.assertEquals(3, doc.get("timestamp"));
- }
- //Get All Latest Guid Testcase
- {
- List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new TypeReference<List<GetRequest>>() {
- });
- Iterator<Document> response = dao.getAllLatest(request).iterator();
- Document bro2 = response.next();
- Assert.assertEquals("bro_2", bro2.getDocument().get("guid"));
- Assert.assertEquals("bro", bro2.getDocument().get("source:type"));
- Document snort2 = response.next();
- Assert.assertEquals("bro_1", snort2.getDocument().get("guid"));
- Assert.assertEquals("bro", snort2.getDocument().get("source:type"));
- Assert.assertFalse(response.hasNext());
- }
- //Filter test case
- {
- SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(3, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
- Assert.assertEquals(9, results.get(0).getSource().get("timestamp"));
- Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
- Assert.assertEquals(7, results.get(1).getSource().get("timestamp"));
- Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
- Assert.assertEquals(1, results.get(2).getSource().get("timestamp"));
- }
- //Sort test case
- {
- SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- for(int i = 8001;i < 8011;++i) {
- Assert.assertEquals(i, results.get(i-8001).getSource().get("ip_src_port"));
- }
+ public void all_query_returns_all_results() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(allQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, results.size());
+ for(int i = 0;i < 5;++i) {
+ Assert.assertEquals("snort", results.get(i).getSource().get("source:type"));
+ Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
}
- //Sort descending with missing fields
- {
- SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, results.size());
-
- // validate sorted order - there are only 2 with a 'threat:triage:score'
- Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
- Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
-
- // the remaining are missing the 'threat:triage:score' and should be sorted last
- Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(8).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(9).getSource().containsKey("threat:triage:score"));
+ for (int i = 5; i < 10; ++i) {
+ Assert.assertEquals("bro", results.get(i).getSource().get("source:type"));
+ Assert.assertEquals(10 - i, results.get(i).getSource().get("timestamp"));
}
- //Sort ascending with missing fields
- {
- SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(10, results.size());
-
- // the remaining are missing the 'threat:triage:score' and should be sorted last
- Assert.assertFalse(results.get(0).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(1).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(2).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(3).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(4).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(5).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(6).getSource().containsKey("threat:triage:score"));
- Assert.assertFalse(results.get(7).getSource().containsKey("threat:triage:score"));
-
- // validate sorted order - there are only 2 with a 'threat:triage:score'
- Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
- Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
- }
- //pagination test case
- {
- SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- Assert.assertEquals(3, results.size());
- Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
- Assert.assertEquals(6, results.get(0).getSource().get("timestamp"));
- Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
- Assert.assertEquals(5, results.get(1).getSource().get("timestamp"));
- Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
- Assert.assertEquals(4, results.get(2).getSource().get("timestamp"));
- }
- //Index query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(5, response.getTotal());
- List<SearchResult> results = response.getResults();
- for(int i = 5,j=0;i > 0;i--,j++) {
- Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
- Assert.assertEquals(i, results.get(j).getSource().get("timestamp"));
- }
- }
- //Facet query including all field types
- {
- SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(12, response.getTotal());
-
- Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
- Assert.assertEquals(8, facetCounts.size());
-
- // source:type
- Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
- Assert.assertEquals(2, sourceTypeCounts.size());
- Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
- Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
-
- // ip_src_addr
- Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr");
- Assert.assertEquals(8, ipSrcAddrCounts.size());
- Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7"));
- Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8"));
-
- // ip_src_port
- Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port");
- Assert.assertEquals(10, ipSrcPortCounts.size());
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009"));
- Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010"));
-
- // long_field
- Map<String, Long> longFieldCounts = facetCounts.get("long_field");
- Assert.assertEquals(2, longFieldCounts.size());
- Assert.assertEquals(new Long(8), longFieldCounts.get("10000"));
- Assert.assertEquals(new Long(2), longFieldCounts.get("20000"));
-
- // timestamp
- Map<String, Long> timestampCounts = facetCounts.get("timestamp");
- Assert.assertEquals(10, timestampCounts.size());
- Assert.assertEquals(new Long(1), timestampCounts.get("1"));
- Assert.assertEquals(new Long(1), timestampCounts.get("2"));
- Assert.assertEquals(new Long(1), timestampCounts.get("3"));
- Assert.assertEquals(new Long(1), timestampCounts.get("4"));
- Assert.assertEquals(new Long(1), timestampCounts.get("5"));
- Assert.assertEquals(new Long(1), timestampCounts.get("6"));
- Assert.assertEquals(new Long(1), timestampCounts.get("7"));
- Assert.assertEquals(new Long(1), timestampCounts.get("8"));
- Assert.assertEquals(new Long(1), timestampCounts.get("9"));
- Assert.assertEquals(new Long(1), timestampCounts.get("10"));
-
- // latitude
- Map<String, Long> latitudeCounts = facetCounts.get("latitude");
- Assert.assertEquals(2, latitudeCounts.size());
- List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet());
- Collections.sort(latitudeKeys);
- Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001);
- Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
- Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
- Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
-
- // score
- Map<String, Long> scoreFieldCounts = facetCounts.get("score");
- Assert.assertEquals(4, scoreFieldCounts.size());
- List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
- Collections.sort(scoreFieldKeys);
- Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
- Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
- Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
- Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
- Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
- Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
- Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
- Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
-
- // is_alert
- Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
- Assert.assertEquals(2, isAlertCounts.size());
- Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
- Assert.assertEquals(new Long(4), isAlertCounts.get("false"));
+ }
+
+ @Test
+ public void find_one_guid() throws Exception {
+ GetRequest request = JSONUtils.INSTANCE.load(findOneGuidQuery, GetRequest.class);
+ Optional<Map<String, Object>> response = dao.getLatestResult(request);
+ Assert.assertTrue(response.isPresent());
+ Map<String, Object> doc = response.get();
+ Assert.assertEquals("bro", doc.get("source:type"));
+ Assert.assertEquals(3, doc.get("timestamp"));
+ }
+
+ @Test
+ public void get_all_latest_guid() throws Exception {
+ List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){});
+ Map<String, Document> docs = new HashMap<>();
+
+ for(Document doc : dao.getAllLatest(request)) {
+ docs.put(doc.getGuid(), doc);
}
- //Bad facet query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class);
- try {
- dao.search(request);
- Assert.fail("Exception expected, but did not come.");
- }
- catch(InvalidSearchException ise) {
- // success
- }
+ Assert.assertEquals(2, docs.size());
+ Assert.assertTrue(docs.keySet().contains("bro_1"));
+ Assert.assertTrue(docs.keySet().contains("snort_2"));
+ Assert.assertEquals("bro", docs.get("bro_1").getDocument().get("source:type"));
+ Assert.assertEquals("snort", docs.get("snort_2").getDocument().get("source:type"));
+ }
+
+ @Test
+ public void filter_query_filters_results() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(filterQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(3, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+ Assert.assertEquals(9, results.get(0).getSource().get("timestamp"));
+ Assert.assertEquals("snort", results.get(1).getSource().get("source:type"));
+ Assert.assertEquals(7, results.get(1).getSource().get("timestamp"));
+ Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+ Assert.assertEquals(1, results.get(2).getSource().get("timestamp"));
+ }
+
+ @Test
+ public void sort_query_sorts_results_ascending() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for (int i = 8001; i < 8011; ++i) {
+ Assert.assertEquals(i, results.get(i - 8001).getSource().get("ip_src_port"));
}
- //Disabled facet query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertNull(response.getFacetCounts());
+ }
+
+ @Test
+ public void sort_ascending_with_missing_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortAscendingWithMissingFields, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, results.size());
+
+ // the remaining are missing the 'threat:triage:score' and should be sorted last
+ for (int i = 0; i < 8; i++) {
+ Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
}
- //Exceeded maximum results query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
- try {
- dao.search(request);
- Assert.fail("Exception expected, but did not come.");
- }
- catch(InvalidSearchException ise) {
- Assert.assertEquals("Search result size must be less than 100", ise.getMessage());
- }
+
+ // validate sorted order - there are only 2 with a 'threat:triage:score'
+ Assert.assertEquals("10", results.get(8).getSource().get("threat:triage:score"));
+ Assert.assertEquals("20", results.get(9).getSource().get("threat:triage:score"));
+ }
+
+ @Test
+ public void sort_descending_with_missing_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortDescendingWithMissingFields, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(10, results.size());
+
+ // validate sorted order - there are only 2 with a 'threat:triage:score'
+ Assert.assertEquals("20", results.get(0).getSource().get("threat:triage:score"));
+ Assert.assertEquals("10", results.get(1).getSource().get("threat:triage:score"));
+
+ // the remaining are missing the 'threat:triage:score' and should be sorted last
+ for (int i = 2; i < 10; i++) {
+ Assert.assertFalse(results.get(i).getSource().containsKey("threat:triage:score"));
}
++<<<<<<< HEAD
+ }
+
+ @Test
+ public void results_are_paginated() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(paginationQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ Assert.assertEquals(3, results.size());
+ Assert.assertEquals("snort", results.get(0).getSource().get("source:type"));
+ Assert.assertEquals(6, results.get(0).getSource().get("timestamp"));
+ Assert.assertEquals("bro", results.get(1).getSource().get("source:type"));
+ Assert.assertEquals(5, results.get(1).getSource().get("timestamp"));
+ Assert.assertEquals("bro", results.get(2).getSource().get("source:type"));
+ Assert.assertEquals(4, results.get(2).getSource().get("timestamp"));
+ }
+
+ @Test
+ public void returns_results_only_for_specified_indices() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(indexQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(5, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for (int i = 5, j = 0; i > 0; i--, j++) {
+ Assert.assertEquals("bro", results.get(j).getSource().get("source:type"));
+ Assert.assertEquals(i, results.get(j).getSource().get("timestamp"));
++=======
+ // getColumnMetadata with multiple indices
+ {
+ Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("searchintegrationtest_bro", "searchintegrationtest_snort"));
+ Assert.assertEquals(15, fieldTypes.size());
+ Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid"));
+ Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
+ Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
+ Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
+ Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
+ Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
+ Assert.assertEquals(FieldType.STRING, fieldTypes.get("bro_field"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
++>>>>>>> upstream/feature/METRON-1344-test-infrastructure
}
+ }
+
+ @Test
+ public void facet_query_yields_field_types() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(facetQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ Map<String, Map<String, Long>> facetCounts = response.getFacetCounts();
+ Assert.assertEquals(8, facetCounts.size());
+ Map<String, Long> sourceTypeCounts = facetCounts.get("source:type");
+ Assert.assertEquals(2, sourceTypeCounts.size());
+ Assert.assertEquals(new Long(5), sourceTypeCounts.get("bro"));
+ Assert.assertEquals(new Long(5), sourceTypeCounts.get("snort"));
+ Map<String, Long> ipSrcAddrCounts = facetCounts.get("ip_src_addr");
+ Assert.assertEquals(8, ipSrcAddrCounts.size());
+ Assert.assertEquals(new Long(3), ipSrcAddrCounts.get("192.168.1.1"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.2"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.3"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.4"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.5"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.6"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.7"));
+ Assert.assertEquals(new Long(1), ipSrcAddrCounts.get("192.168.1.8"));
+ Map<String, Long> ipSrcPortCounts = facetCounts.get("ip_src_port");
+ Assert.assertEquals(10, ipSrcPortCounts.size());
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8001"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8002"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8003"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8004"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8005"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8006"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8007"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8008"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8009"));
+ Assert.assertEquals(new Long(1), ipSrcPortCounts.get("8010"));
+ Map<String, Long> longFieldCounts = facetCounts.get("long_field");
+ Assert.assertEquals(2, longFieldCounts.size());
+ Assert.assertEquals(new Long(8), longFieldCounts.get("10000"));
+ Assert.assertEquals(new Long(2), longFieldCounts.get("20000"));
+ Map<String, Long> timestampCounts = facetCounts.get("timestamp");
+ Assert.assertEquals(10, timestampCounts.size());
+ Assert.assertEquals(new Long(1), timestampCounts.get("1"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("2"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("3"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("4"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("5"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("6"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("7"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("8"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("9"));
+ Assert.assertEquals(new Long(1), timestampCounts.get("10"));
+ Map<String, Long> latitudeCounts = facetCounts.get("latitude");
+ Assert.assertEquals(2, latitudeCounts.size());
+ List<String> latitudeKeys = new ArrayList<>(latitudeCounts.keySet());
+ Collections.sort(latitudeKeys);
+ Assert.assertEquals(48.0001, Double.parseDouble(latitudeKeys.get(0)), 0.00001);
+ Assert.assertEquals(48.5839, Double.parseDouble(latitudeKeys.get(1)), 0.00001);
+ Assert.assertEquals(new Long(2), latitudeCounts.get(latitudeKeys.get(0)));
+ Assert.assertEquals(new Long(8), latitudeCounts.get(latitudeKeys.get(1)));
+ Map<String, Long> scoreFieldCounts = facetCounts.get("score");
+ Assert.assertEquals(4, scoreFieldCounts.size());
+ List<String> scoreFieldKeys = new ArrayList<>(scoreFieldCounts.keySet());
+ Collections.sort(scoreFieldKeys);
+ Assert.assertEquals(10.0, Double.parseDouble(scoreFieldKeys.get(0)), 0.00001);
+ Assert.assertEquals(20.0, Double.parseDouble(scoreFieldKeys.get(1)), 0.00001);
+ Assert.assertEquals(50.0, Double.parseDouble(scoreFieldKeys.get(2)), 0.00001);
+ Assert.assertEquals(98.0, Double.parseDouble(scoreFieldKeys.get(3)), 0.00001);
+ Assert.assertEquals(new Long(4), scoreFieldCounts.get(scoreFieldKeys.get(0)));
+ Assert.assertEquals(new Long(2), scoreFieldCounts.get(scoreFieldKeys.get(1)));
+ Assert.assertEquals(new Long(3), scoreFieldCounts.get(scoreFieldKeys.get(2)));
+ Assert.assertEquals(new Long(1), scoreFieldCounts.get(scoreFieldKeys.get(3)));
+ Map<String, Long> isAlertCounts = facetCounts.get("is_alert");
+ Assert.assertEquals(2, isAlertCounts.size());
+ Assert.assertEquals(new Long(6), isAlertCounts.get("true"));
+ Assert.assertEquals(new Long(4), isAlertCounts.get("false"));
+ }
+
+ @Test
+ public void bad_facet_query_throws_exception() throws Exception {
+ thrown.expect(InvalidSearchException.class);
+ thrown.expectMessage("Failed to execute search");
+ SearchRequest request = JSONUtils.INSTANCE.load(badFacetQuery, SearchRequest.class);
+ dao.search(request);
+ }
+
+ @Test
+ public void disabled_facet_query_returns_null_count() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(disabledFacetQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertNull(response.getFacetCounts());
+ }
+
+ @Test
+ public void exceeding_max_resulsts_throws_exception() throws Exception {
+ thrown.expect(InvalidSearchException.class);
+ thrown.expectMessage("Search result size must be less than 100");
+ SearchRequest request = JSONUtils.INSTANCE.load(exceededMaxResultsQuery, SearchRequest.class);
+ dao.search(request);
+ }
+
+ @Test
+ public void returns_column_data_for_multiple_indices() throws Exception {
+ Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort"));
+ Assert.assertEquals(15, fieldTypes.size());
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
+ Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
+ Assert.assertEquals(FieldType.DATE, fieldTypes.get("timestamp"));
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("latitude"));
+ Assert.assertEquals(FieldType.DOUBLE, fieldTypes.get("score"));
+ Assert.assertEquals(FieldType.BOOLEAN, fieldTypes.get("is_alert"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("location_point"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
+ //NOTE: This is because the field is in both bro and snort and they have different types.
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.FLOAT, fieldTypes.get("threat:triage:score"));
+ Assert.assertEquals(FieldType.OTHER, fieldTypes.get("alert"));
+ }
+
+ @Test
+ public void returns_column_metadata_for_specified_indices() throws Exception {
// getColumnMetadata with only bro
{
- Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro"));
+ Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_bro"));
Assert.assertEquals(13, fieldTypes.size());
- Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("bro_field"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@@ -708,12 -738,11 +738,12 @@@
}
// getColumnMetadata with only snort
{
- Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort"));
+ Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_snort"));
Assert.assertEquals(14, fieldTypes.size());
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field"));
- Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid"));
- Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type"));
+ Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("duplicate_name_field"));
+ Assert.assertEquals(FieldType.KEYWORD, fieldTypes.get("guid"));
+ Assert.assertEquals(FieldType.TEXT, fieldTypes.get("source:type"));
Assert.assertEquals(FieldType.IP, fieldTypes.get("ip_src_addr"));
Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("ip_src_port"));
Assert.assertEquals(FieldType.LONG, fieldTypes.get("long_field"));
@@@ -727,211 -756,207 +757,211 @@@
}
// getColumnMetadata with an index that doesn't exist
{
- Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex"));
+ Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_someindex"));
Assert.assertEquals(0, fieldTypes.size());
}
- //Fields query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(10, response.getTotal());
- List<SearchResult> results = response.getResults();
- for(int i = 0;i < 5;++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertEquals(1, source.size());
- Assert.assertNotNull(source.get("ip_src_addr"));
- }
- for(int i = 5;i < 10;++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertEquals(1, source.size());
- Assert.assertNotNull(source.get("ip_src_addr"));
- }
- }
- //Meta Alerts Fields query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(metaAlertsFieldQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(2, response.getTotal());
- List<SearchResult> results = response.getResults();
- for (int i = 0;i < 2;++i) {
- Map<String, Object> source = results.get(i).getSource();
- Assert.assertEquals(1, source.size());
- Assert.assertEquals(source.get("guid"), "meta_" + (i + 1));
- }
- }
- //No results fields query
- {
- SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
- SearchResponse response = dao.search(request);
- Assert.assertEquals(0, response.getTotal());
- }
- // Group by test case, default order is count descending
- {
- GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
- GroupResponse response = dao.group(request);
- Assert.assertEquals("is_alert", response.getGroupedBy());
- List<GroupResult> isAlertGroups = response.getGroupResults();
- Assert.assertEquals(2, isAlertGroups.size());
-
- // isAlert == true group
- GroupResult trueGroup = isAlertGroups.get(0);
- Assert.assertEquals("true", trueGroup.getKey());
- Assert.assertEquals(6, trueGroup.getTotal());
- Assert.assertEquals("latitude", trueGroup.getGroupedBy());
- Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001);
- List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults();
- Assert.assertEquals(2, trueLatitudeGroups.size());
-
- // isAlert == true && latitude == 48.5839 group
- GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0);
- Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001);
- Assert.assertEquals(5, trueLatitudeGroup2.getTotal());
- Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001);
-
- // isAlert == true && latitude == 48.0001 group
- GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1);
- Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001);
- Assert.assertEquals(1, trueLatitudeGroup1.getTotal());
- Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001);
-
- // isAlert == false group
- GroupResult falseGroup = isAlertGroups.get(1);
- Assert.assertEquals("false", falseGroup.getKey());
- Assert.assertEquals("latitude", falseGroup.getGroupedBy());
- Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001);
- List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults();
- Assert.assertEquals(2, falseLatitudeGroups.size());
-
- // isAlert == false && latitude == 48.5839 group
- GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0);
- Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001);
- Assert.assertEquals(3, falseLatitudeGroup2.getTotal());
- Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001);
-
- // isAlert == false && latitude == 48.0001 group
- GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1);
- Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001);
- Assert.assertEquals(1, falseLatitudeGroup1.getTotal());
- Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001);
- }
- // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
- {
- GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
- GroupResponse response = dao.group(request);
- Assert.assertEquals("is_alert", response.getGroupedBy());
- List<GroupResult> isAlertGroups = response.getGroupResults();
- Assert.assertEquals(2, isAlertGroups.size());
-
- // isAlert == false group
- GroupResult falseGroup = isAlertGroups.get(0);
- Assert.assertEquals(4, falseGroup.getTotal());
- Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy());
- List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults();
- Assert.assertEquals(4, falseIpSrcAddrGroups.size());
-
- // isAlert == false && ip_src_addr == 192.168.1.8 group
- GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0);
- Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey());
- Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal());
- Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy());
- Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults());
-
- // isAlert == false && ip_src_addr == 192.168.1.7 group
- GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1);
- Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey());
- Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal());
- Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy());
- Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults());
-
- // isAlert == false && ip_src_addr == 192.168.1.6 group
- GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2);
- Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey());
- Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal());
- Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy());
- Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults());
-
- // isAlert == false && ip_src_addr == 192.168.1.2 group
- GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3);
- Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey());
- Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal());
- Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy());
- Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults());
-
- // isAlert == false group
- GroupResult trueGroup = isAlertGroups.get(1);
- Assert.assertEquals(6, trueGroup.getTotal());
- Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy());
- List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults();
- Assert.assertEquals(4, trueIpSrcAddrGroups.size());
-
- // isAlert == false && ip_src_addr == 192.168.1.5 group
- GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0);
- Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey());
- Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal());
- Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy());
- Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults());
-
- // isAlert == false && ip_src_addr == 192.168.1.4 group
- GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1);
- Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey());
- Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal());
- Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy());
- Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults());
-
- // isAlert == false && ip_src_addr == 192.168.1.3 group
- GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2);
- Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey());
- Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal());
- Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy());
- Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults());
-
- // isAlert == false && ip_src_addr == 192.168.1.1 group
- GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3);
- Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey());
- Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal());
- Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy());
- Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults());
- }
- //Bad group query
- {
- GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class);
- try {
- dao.group(request);
- Assert.fail("Exception expected, but did not come.");
- }
- catch(InvalidSearchException ise) {
- // success
- }
- }
- //Group by IP query
- {
- {
- GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class);
- GroupResponse response = dao.group(request);
+ }
+
+
+ @Test
+ public void group_by_ip_query() throws Exception {
+ GroupRequest request = JSONUtils.INSTANCE.load(groupByIpQuery, GroupRequest.class);
+ GroupResponse response = dao.group(request);
+
+ // expect only 1 group for 'ip_src_addr'
+ Assert.assertEquals("ip_src_addr", response.getGroupedBy());
+
+ // there are 8 different 'ip_src_addr' values
+ List<GroupResult> groups = response.getGroupResults();
+ Assert.assertEquals(8, groups.size());
+
+ // expect dotted-decimal notation in descending order
+ Assert.assertEquals("192.168.1.8", groups.get(0).getKey());
+ Assert.assertEquals("192.168.1.7", groups.get(1).getKey());
+ Assert.assertEquals("192.168.1.6", groups.get(2).getKey());
+ Assert.assertEquals("192.168.1.5", groups.get(3).getKey());
+ Assert.assertEquals("192.168.1.4", groups.get(4).getKey());
+ Assert.assertEquals("192.168.1.3", groups.get(5).getKey());
+ Assert.assertEquals("192.168.1.2", groups.get(6).getKey());
+ Assert.assertEquals("192.168.1.1", groups.get(7).getKey());
+ }
- // expect only 1 group for 'ip_src_addr'
- Assert.assertEquals("ip_src_addr", response.getGroupedBy());
+ @Test
+ public void no_results_returned_when_query_does_not_match() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(noResultsFieldsQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(0, response.getTotal());
+ }
+
+ @Test
+ public void group_by_returns_results_in_groups() throws Exception {
+ // Group by test case, default order is count descending
+ GroupRequest request = JSONUtils.INSTANCE.load(groupByQuery, GroupRequest.class);
+ GroupResponse response = dao.group(request);
+ Assert.assertEquals("is_alert", response.getGroupedBy());
+ List<GroupResult> isAlertGroups = response.getGroupResults();
+ Assert.assertEquals(2, isAlertGroups.size());
+
+ // isAlert == true group
+ GroupResult trueGroup = isAlertGroups.get(0);
+ Assert.assertEquals("true", trueGroup.getKey());
+ Assert.assertEquals(6, trueGroup.getTotal());
+ Assert.assertEquals("latitude", trueGroup.getGroupedBy());
+ Assert.assertEquals(198.0, trueGroup.getScore(), 0.00001);
+ List<GroupResult> trueLatitudeGroups = trueGroup.getGroupResults();
+ Assert.assertEquals(2, trueLatitudeGroups.size());
+
+
+ // isAlert == true && latitude == 48.5839 group
+ GroupResult trueLatitudeGroup2 = trueLatitudeGroups.get(0);
+ Assert.assertEquals(48.5839, Double.parseDouble(trueLatitudeGroup2.getKey()), 0.00001);
+ Assert.assertEquals(5, trueLatitudeGroup2.getTotal());
+ Assert.assertEquals(148.0, trueLatitudeGroup2.getScore(), 0.00001);
+
+ // isAlert == true && latitude == 48.0001 group
+ GroupResult trueLatitudeGroup1 = trueLatitudeGroups.get(1);
+ Assert.assertEquals(48.0001, Double.parseDouble(trueLatitudeGroup1.getKey()), 0.00001);
+ Assert.assertEquals(1, trueLatitudeGroup1.getTotal());
+ Assert.assertEquals(50.0, trueLatitudeGroup1.getScore(), 0.00001);
+
+ // isAlert == false group
+ GroupResult falseGroup = isAlertGroups.get(1);
+ Assert.assertEquals("false", falseGroup.getKey());
+ Assert.assertEquals("latitude", falseGroup.getGroupedBy());
+ Assert.assertEquals(130.0, falseGroup.getScore(), 0.00001);
+ List<GroupResult> falseLatitudeGroups = falseGroup.getGroupResults();
+ Assert.assertEquals(2, falseLatitudeGroups.size());
+
+ // isAlert == false && latitude == 48.5839 group
+ GroupResult falseLatitudeGroup2 = falseLatitudeGroups.get(0);
+ Assert.assertEquals(48.5839, Double.parseDouble(falseLatitudeGroup2.getKey()), 0.00001);
+ Assert.assertEquals(3, falseLatitudeGroup2.getTotal());
+ Assert.assertEquals(80.0, falseLatitudeGroup2.getScore(), 0.00001);
+
+ // isAlert == false && latitude == 48.0001 group
+ GroupResult falseLatitudeGroup1 = falseLatitudeGroups.get(1);
+ Assert.assertEquals(48.0001, Double.parseDouble(falseLatitudeGroup1.getKey()), 0.00001);
+ Assert.assertEquals(1, falseLatitudeGroup1.getTotal());
+ Assert.assertEquals(50.0, falseLatitudeGroup1.getScore(), 0.00001);
+ }
- // there are 8 different 'ip_src_addr' values
- List<GroupResult> groups = response.getGroupResults();
- Assert.assertEquals(8, groups.size());
+ @Test
+ public void group_by_returns_results_in_sorted_groups() throws Exception {
+ // Group by with sorting test case where is_alert is sorted by count ascending and ip_src_addr is sorted by term descending
+ GroupRequest request = JSONUtils.INSTANCE.load(sortedGroupByQuery, GroupRequest.class);
+ GroupResponse response = dao.group(request);
+ Assert.assertEquals("is_alert", response.getGroupedBy());
+ List<GroupResult> isAlertGroups = response.getGroupResults();
+ Assert.assertEquals(2, isAlertGroups.size());
+
+ // isAlert == false group
+ GroupResult falseGroup = isAlertGroups.get(0);
+ Assert.assertEquals(4, falseGroup.getTotal());
+ Assert.assertEquals("ip_src_addr", falseGroup.getGroupedBy());
+ List<GroupResult> falseIpSrcAddrGroups = falseGroup.getGroupResults();
+ Assert.assertEquals(4, falseIpSrcAddrGroups.size());
+
+ // isAlert == false && ip_src_addr == 192.168.1.8 group
+ GroupResult falseIpSrcAddrGroup1 = falseIpSrcAddrGroups.get(0);
+ Assert.assertEquals("192.168.1.8", falseIpSrcAddrGroup1.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup1.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup1.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup1.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.7 group
+ GroupResult falseIpSrcAddrGroup2 = falseIpSrcAddrGroups.get(1);
+ Assert.assertEquals("192.168.1.7", falseIpSrcAddrGroup2.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup2.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup2.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup2.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.6 group
+ GroupResult falseIpSrcAddrGroup3 = falseIpSrcAddrGroups.get(2);
+ Assert.assertEquals("192.168.1.6", falseIpSrcAddrGroup3.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup3.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup3.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup3.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.2 group
+ GroupResult falseIpSrcAddrGroup4 = falseIpSrcAddrGroups.get(3);
+ Assert.assertEquals("192.168.1.2", falseIpSrcAddrGroup4.getKey());
+ Assert.assertEquals(1, falseIpSrcAddrGroup4.getTotal());
+ Assert.assertNull(falseIpSrcAddrGroup4.getGroupedBy());
+ Assert.assertNull(falseIpSrcAddrGroup4.getGroupResults());
+
+ // isAlert == false group
+ GroupResult trueGroup = isAlertGroups.get(1);
+ Assert.assertEquals(6, trueGroup.getTotal());
+ Assert.assertEquals("ip_src_addr", trueGroup.getGroupedBy());
+ List<GroupResult> trueIpSrcAddrGroups = trueGroup.getGroupResults();
+ Assert.assertEquals(4, trueIpSrcAddrGroups.size());
+
+ // isAlert == false && ip_src_addr == 192.168.1.5 group
+ GroupResult trueIpSrcAddrGroup1 = trueIpSrcAddrGroups.get(0);
+ Assert.assertEquals("192.168.1.5", trueIpSrcAddrGroup1.getKey());
+ Assert.assertEquals(1, trueIpSrcAddrGroup1.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup1.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup1.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.4 group
+ GroupResult trueIpSrcAddrGroup2 = trueIpSrcAddrGroups.get(1);
+ Assert.assertEquals("192.168.1.4", trueIpSrcAddrGroup2.getKey());
+ Assert.assertEquals(1, trueIpSrcAddrGroup2.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup2.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup2.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.3 group
+ GroupResult trueIpSrcAddrGroup3 = trueIpSrcAddrGroups.get(2);
+ Assert.assertEquals("192.168.1.3", trueIpSrcAddrGroup3.getKey());
+ Assert.assertEquals(1, trueIpSrcAddrGroup3.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup3.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup3.getGroupResults());
+
+ // isAlert == false && ip_src_addr == 192.168.1.1 group
+ GroupResult trueIpSrcAddrGroup4 = trueIpSrcAddrGroups.get(3);
+ Assert.assertEquals("192.168.1.1", trueIpSrcAddrGroup4.getKey());
+ Assert.assertEquals(3, trueIpSrcAddrGroup4.getTotal());
+ Assert.assertNull(trueIpSrcAddrGroup4.getGroupedBy());
+ Assert.assertNull(trueIpSrcAddrGroup4.getGroupResults());
+ }
- // expect dotted-decimal notation in descending order
- Assert.assertEquals("192.168.1.8", groups.get(0).getKey());
- Assert.assertEquals("192.168.1.7", groups.get(1).getKey());
- Assert.assertEquals("192.168.1.6", groups.get(2).getKey());
- Assert.assertEquals("192.168.1.5", groups.get(3).getKey());
- Assert.assertEquals("192.168.1.4", groups.get(4).getKey());
- Assert.assertEquals("192.168.1.3", groups.get(5).getKey());
- Assert.assertEquals("192.168.1.2", groups.get(6).getKey());
- Assert.assertEquals("192.168.1.1", groups.get(7).getKey());
- }
+ @Test
+ public void throws_exception_on_aggregation_queries_on_non_string_non_numeric_fields()
+ throws Exception {
+ thrown.expect(InvalidSearchException.class);
+ thrown.expectMessage("Failed to execute search");
+ GroupRequest request = JSONUtils.INSTANCE.load(badGroupQuery, GroupRequest.class);
+ dao.group(request);
+ }
+ @Test
+ public void queries_fields() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(fieldsQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(10, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for (int i = 0; i < 5; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertNotNull(source.get("ip_src_addr"));
+ }
+ for (int i = 5; i < 10; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertNotNull(source.get("ip_src_addr"));
+ }
+ }
+ @Test
+ public void sort_by_guid() throws Exception {
+ SearchRequest request = JSONUtils.INSTANCE.load(sortByGuidQuery, SearchRequest.class);
+ SearchResponse response = dao.search(request);
+ Assert.assertEquals(5, response.getTotal());
+ List<SearchResult> results = response.getResults();
+ for (int i = 0; i < 5; ++i) {
+ Map<String, Object> source = results.get(i).getSource();
+ Assert.assertEquals(1, source.size());
+ Assert.assertEquals(source.get("guid"), "bro_" + (i + 1));
}
}
[25/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml b/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml
new file mode 100644
index 0000000..fc6c094
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: "Allocate {{ swapspace }} for swap space"
+ shell: fallocate -l {{ swapspace }} {{ swapfile }}
+
+- name: "Permissioning {{ swapfile }}"
+ file:
+ path: "{{ swapfile }}"
+ mode: "600"
+
+- name: "Setup swap space"
+ shell: mkswap {{ swapfile }}
+
+- name: "Enable swap space"
+ shell: swapon {{ swapfile }}
+
+- name: "Enable swap on boot"
+ lineinfile:
+ dest: "/etc/fstab"
+ line: "{{ swapfile }} none swap sw 0 0"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
new file mode 100644
index 0000000..f7d108c
--- /dev/null
+++ b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: "Is swap space enabled?"
+ shell: swapon -s | grep {{ swapfile }}
+ register: swapcheck
+ failed_when: swapcheck.rc != 0 and swapcheck.rc != 1
+
+- debug: msg="Swap space is already enabled"
+ when: swapcheck.rc == 0
+
+- debug: msg="Swap space is NOT yet enabled"
+ when: swapcheck.rc == 1
+
+- include: enable-swap.yml
+ static: no
+ when: swapcheck.rc != 0
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/epel/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/epel/tasks/main.yml b/metron-deployment/ansible/roles/epel/tasks/main.yml
new file mode 100644
index 0000000..5fecc6c
--- /dev/null
+++ b/metron-deployment/ansible/roles/epel/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install EPEL repository
+ yum: name=epel-release
+ when: ansible_distribution == "CentOS"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/defaults/main.yml b/metron-deployment/ansible/roles/fastcapa/defaults/main.yml
new file mode 100644
index 0000000..d586046
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/defaults/main.yml
@@ -0,0 +1,42 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# dpdk
+dpdk_home: "/usr/local/dpdk"
+dpdk_version: "17.08"
+dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
+dpdk_src_url: "http://fast.dpdk.org/rel/dpdk-{{ dpdk_version }}.tar.xz"
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+extra_cflags: -g
+
+# fastcapa
+fastcapa_work_dir: /root/fastcapa
+fastcapa_build_dir: "{{ fastcapa_work_dir }}/build/app/"
+fastcapa_prefix: /usr/local/bin
+fastcapa_ld_library_path: /usr/local/lib
+fastcapa_bin: fastcapa
+
+# fastcapa settings
+fastcapa_portmask: 0x01
+fastcapa_kafka_config: /etc/fastcapa.conf
+fastcapa_topic: pcap
+fastcapa_rx_burst_size: 32
+fastcapa_tx_burst_size: 256
+fastcapa_nb_rx_desc: 1024
+fastcapa_nb_rx_queue: 1
+fastcapa_tx_ring_size: 2048
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/meta/main.yml b/metron-deployment/ansible/roles/fastcapa/meta/main.yml
new file mode 100644
index 0000000..d253e88
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml b/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml
new file mode 100644
index 0000000..06f1526
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/tasks/debug.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install debug utilities
+ yum: name=yum-utils
+ tags:
+ - debug
+
+ - name: Install debug symbols
+ shell: debuginfo-install -y glibc glib2 zlib
+ tags:
+ - debug
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml b/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml
new file mode 100644
index 0000000..5955702
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/tasks/dependencies.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Install dependencies
+ yum: name={{ item }}
+ with_items:
+ - "@Development tools"
+ - pciutils
+ - net-tools
+ - glib2
+ - glib2-devel
+ - git
+ - numactl-devel
+
+ #
+ # install prerequisite packages and the latest kernel headers. need to
+ # ensure that the kernel headers match the current running kernel version.
+ # if this is not the case, the DPDK build process will fail
+ #
+ - name: Install latest kernel headers and source
+ yum: name={{ item }} state=latest
+ with_items:
+ - kernel
+ - kernel-devel
+ - kernel-headers
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml b/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml
new file mode 100644
index 0000000..a936570
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/tasks/dpdk.yml
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: "Download DPDK version {{ dpdk_version }}"
+ unarchive:
+ src: "{{ dpdk_src_url }}"
+ dest: "/root"
+ creates: "{{ dpdk_sdk }}"
+ copy: no
+
+ - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: "Turn on debug flags"
+ lineinfile:
+ dest: "{{ dpdk_sdk }}/config/common_linuxapp"
+ regexp: 'DEBUG=n'
+ line: 'DEBUG=y'
+ tags:
+ - debug
+
+ - name: "Build DPDK for the target environment: {{ dpdk_target }}"
+ shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
+ args:
+ chdir: "{{ dpdk_sdk }}"
+ creates: "{{ dpdk_home }}"
+
+ - name: Load kernel modules to enable userspace IO
+ shell: "{{ item }}"
+ with_items:
+ - modprobe uio_pci_generic
+ - modprobe vfio-pci
+
+ - name: Bind the device to the loaded kernel module(s)
+ shell: "{{ dpdk_home }}/sbin/dpdk-devbind --force --bind=uio_pci_generic {{ item }}"
+ with_items: "{{ dpdk_device }}"
+
+ - name: Set useful environment variables
+ lineinfile: "dest=/root/.bash_profile line={{ item }}"
+ with_items:
+ - "export RTE_SDK={{ dpdk_sdk }}"
+ - "export RTE_TARGET={{ dpdk_target }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml b/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml
new file mode 100644
index 0000000..b555668
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/tasks/fastcapa.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Distribute fastcapa
+ copy: src=../../../metron-sensors/fastcapa dest={{ fastcapa_work_dir | dirname }} mode=0755
+
+- name: Build fastcapa
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ fastcapa_work_dir }}"
+ with_items:
+ - make
+ environment:
+ RTE_SDK: "{{ dpdk_sdk }}"
+ RTE_TARGET: "{{ dpdk_target }}"
+ LD_LIBRARY_PATH: "{{ fastcapa_ld_library_path }}"
+
+- name: Install fastcapa
+ shell: "cp {{ fastcapa_build_dir }}/{{ fastcapa_bin }} {{ fastcapa_prefix }}"
+ args:
+ chdir: "{{ fastcapa_work_dir }}"
+ creates: "{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
+
+- name: Deploy configuration
+ template: src=fastcapa.conf dest={{ fastcapa_kafka_config }} mode=0755
+
+- name: Deploy service
+ template: src=fastcapa dest=/etc/init.d/ mode=0755
+
+- name: Register the service with systemd
+ shell: systemctl enable fastcapa
+ when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml b/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml
new file mode 100644
index 0000000..cd4abe6
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/tasks/kernel.yml
@@ -0,0 +1,51 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+#
+# DPDK requires specific kernel boot parameters. set the params and reboot
+# the host, if the actual params differ from what is expected.
+#
+---
+ - set_fact:
+ expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
+
+ - name: Check kernel boot parameters
+ shell: "cat /proc/cmdline"
+ register: actual_kernel_params
+
+ - name: Alter kernel boot parameters
+ lineinfile:
+ dest: /etc/default/grub
+ regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
+ line: '\1 {{ expected_kernel_params }}"'
+ backrefs: yes
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Update grub with kernel boot parameters
+ shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Restart for modified kernel params
+ command: shutdown -r now "modified kernel params"
+ async: 0
+ poll: 0
+ ignore_errors: true
+ when: not expected_kernel_params in actual_kernel_params.stdout
+
+ - name: Wait for reboot of '{{ inventory_hostname }}'
+ local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
+ become: false
+ when: not expected_kernel_params in actual_kernel_params.stdout
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/tasks/main.yml b/metron-deployment/ansible/roles/fastcapa/tasks/main.yml
new file mode 100644
index 0000000..b98d557
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/tasks/main.yml
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - include: dependencies.yml
+ - include: kernel.yml
+ - include: dpdk.yml
+ - include: fastcapa.yml
+ - include: debug.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/templates/fastcapa
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/templates/fastcapa b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa
new file mode 100644
index 0000000..f34c603
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa
@@ -0,0 +1,145 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# fastcapa daemon
+# chkconfig: 345 20 80
+# description: Packet capture probe
+# processname: fastcapa
+#
+
+export LD_LIBRARY_PATH="{{ fastcapa_ld_library_path }}"
+
+NAME="fastcapa"
+DESC="Metron network packet capture probe"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGROOT=/var/log/fastcapa
+DAEMONLOG=$LOGROOT/$NAME-stdout.log
+DAEMONERR=$LOGROOT/$NAME-stderr.log
+NOW=`date`
+DAEMON_PATH="/root"
+
+PORT_MASK="{{ fastcapa_portmask }}"
+KAFKA_TOPIC="{{ fastcapa_topic }}"
+KAFKA_CONFIG="{{ fastcapa_kafka_config }}"
+RX_BURST_SIZE="{{ fastcapa_rx_burst_size }}"
+TX_BURST_SIZE="{{ fastcapa_tx_burst_size }}"
+NB_RX_DESC="{{ fastcapa_nb_rx_desc }}"
+NB_RX_QUEUE="{{ fastcapa_nb_rx_queue }}"
+TX_RING_SIZE="{{ fastcapa_tx_ring_size }}"
+
+DAEMON="{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
+DAEMONOPTS+=" "
+DAEMONOPTS+=" -- "
+DAEMONOPTS+="-p $PORT_MASK "
+DAEMONOPTS+="-t $KAFKA_TOPIC "
+DAEMONOPTS+="-c $KAFKA_CONFIG "
+DAEMONOPTS+="-b $RX_BURST_SIZE "
+DAEMONOPTS+="-w $TX_BURST_SIZE "
+DAEMONOPTS+="-d $NB_RX_DESC "
+DAEMONOPTS+="-q $NB_RX_QUEUE "
+DAEMONOPTS+="-x $TX_RING_SIZE "
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+ echo "$NOW: Starting $NAME..." >> $DAEMONLOG
+
+ mkdir -p $LOGROOT
+ touch $DAEMONLOG
+ touch $DAEMONERR
+
+ cd $DAEMON_PATH
+ echo "$DAEMON $DAEMONOPTS >> $DAEMONLOG 2> $DAEMONERR" >> $DAEMONLOG
+
+ if [ -f $PIDFILE ]; then
+ printf "%s\n" "Already running"
+ else
+ PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2> $DAEMONERR & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ while sleep 1
+ echo -n "."
+ kill -0 $PID >/dev/null 2>&1
+ do
+ kill -SIGINT $PID
+ done
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ tail)
+ tail -F $LOGROOT/*
+ ;;
+
+ kill)
+ printf "%-50s" "Force killing $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ while sleep 1
+ echo -n "."
+ kill -0 $PID >/dev/null 2>&1
+ do
+ kill -SIGTERM $PID
+ done
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart|kill|tail}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf
new file mode 100644
index 0000000..7d9eae4
--- /dev/null
+++ b/metron-deployment/ansible/roles/fastcapa/templates/fastcapa.conf
@@ -0,0 +1,67 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# kafka global settings
+#
+[kafka-global]
+
+# initial list of kafka brokers
+metadata.broker.list = {{ kafka_broker_url }}
+
+# identifies the client to kafka
+client.id = metron-fastcapa
+
+# max number of messages allowed on the producer queue
+queue.buffering.max.messages = 1000000
+
+# maximum time, in milliseconds, for buffering data on the producer queue
+queue.buffering.max.ms = 3000
+
+# compression codec = none, gzip or snappy
+compression.codec = snappy
+
+# maximum number of messages batched in one MessageSet (increase for better compression)
+batch.num.messages = 10
+
+# max times to retry sending a failed message set
+message.send.max.retries = 5
+
+# backoff time before retrying a message send
+retry.backoff.ms = 250
+
+# how often statistics are emitted; 0 = never
+statistics.interval.ms = 5000
+
+# only provide delivery reports for failed messages
+delivery.report.only.error = false
+
+#
+# kafka topic settings
+#
+[kafka-topic]
+
+# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
+request.required.acks = 1
+
+# local message timeout. This value is only enforced locally and limits the time a
+# produced message waits for successful delivery. A time of 0 is infinite.
+message.timeout.ms = 10000
+
+# report offset of produced message back to application. The application must be
+# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
+produce.offset.report = false
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/httplib2/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/httplib2/tasks/main.yml b/metron-deployment/ansible/roles/httplib2/tasks/main.yml
new file mode 100644
index 0000000..5502cf4
--- /dev/null
+++ b/metron-deployment/ansible/roles/httplib2/tasks/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install python httplib2 dependency
+ pip:
+ name: httplib2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/java_jdk/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/defaults/main.yml b/metron-deployment/ansible/roles/java_jdk/defaults/main.yml
new file mode 100644
index 0000000..315bc0c
--- /dev/null
+++ b/metron-deployment/ansible/roles/java_jdk/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+java_home: /usr/jdk64/jdk1.8.0_77
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/java_jdk/tasks/main.yml b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
new file mode 100644
index 0000000..999b9c1
--- /dev/null
+++ b/metron-deployment/ansible/roles/java_jdk/tasks/main.yml
@@ -0,0 +1,34 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Check for java at "{{ java_home }}"
+ stat: path="{{ java_home }}"
+ register: jdk_dir
+
+- name: Alternatives link for java
+ alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
+ with_items:
+ - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
+ - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
+ when: jdk_dir.stat.exists
+
+- name: Install openjdk
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - java-1.8.0-openjdk-devel
+ when: not jdk_dir.stat.exists
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml b/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml
new file mode 100644
index 0000000..b0f5a11
--- /dev/null
+++ b/metron-deployment/ansible/roles/kafka-broker/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-broker/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/kafka-broker/meta/main.yml b/metron-deployment/ansible/roles/kafka-broker/meta/main.yml
new file mode 100644
index 0000000..9587e79
--- /dev/null
+++ b/metron-deployment/ansible/roles/kafka-broker/meta/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+dependencies:
+ - libselinux-python
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml b/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml
new file mode 100644
index 0000000..db05cb0
--- /dev/null
+++ b/metron-deployment/ansible/roles/kafka-broker/tasks/main.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Retrieve HDP repository definition
+ get_url:
+ url: "{{ hdp_repo_def }}"
+ dest: /etc/yum.repos.d/hdp.repo
+ mode: 0644
+
+- name: Install kafka
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - kafka
+ - zookeeper-server
+
+- name: Create pid directories
+ file: path={{ item }} state=directory mode=0755
+ with_items:
+ - /var/run/zookeeper
+ - /var/run/kafka
+
+- name: Start zookeeper
+ shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
+
+- name: Start kafka
+ shell: /usr/hdp/current/kafka-broker/bin/kafka start
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-client/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/kafka-client/defaults/main.yml b/metron-deployment/ansible/roles/kafka-client/defaults/main.yml
new file mode 100644
index 0000000..b0f5a11
--- /dev/null
+++ b/metron-deployment/ansible/roles/kafka-client/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/kafka-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/kafka-client/tasks/main.yml b/metron-deployment/ansible/roles/kafka-client/tasks/main.yml
new file mode 100644
index 0000000..1674225
--- /dev/null
+++ b/metron-deployment/ansible/roles/kafka-client/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Retrieve HDP repository definition
+ get_url:
+ url: "{{ hdp_repo_def }}"
+ dest: /etc/yum.repos.d/hdp.repo
+ mode: 0644
+
+ - name: Install kafka
+ yum:
+ name: kafka
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/librdkafka/defaults/main.yml b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
new file mode 100644
index 0000000..063c22f
--- /dev/null
+++ b/metron-deployment/ansible/roles/librdkafka/defaults/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+librdkafka_version: 0.9.4
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml b/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml
new file mode 100644
index 0000000..72ff907
--- /dev/null
+++ b/metron-deployment/ansible/roles/librdkafka/tasks/dependencies.yml
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install prerequisites
+ yum: name={{ item }}
+ with_items:
+ - cmake
+ - make
+ - gcc
+ - gcc-c++
+ - flex
+ - bison
+ - libpcap
+ - libpcap-devel
+ - openssl-devel
+ - python-devel
+ - swig
+ - zlib-devel
+ - perl
+ - cyrus-sasl
+ - cyrus-sasl-devel
+ - cyrus-sasl-gssapi
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml b/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml
new file mode 100644
index 0000000..a7971d6
--- /dev/null
+++ b/metron-deployment/ansible/roles/librdkafka/tasks/librdkafka.yml
@@ -0,0 +1,39 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Download librdkafka
+ get_url:
+ url: "{{ librdkafka_url }}"
+ dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+
+- name: Extract librdkafka tarball
+ unarchive:
+ src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
+ dest: /tmp
+ copy: no
+ creates: "/tmp/librdkafka-{{ librdkafka_version }}"
+
+- name: Compile and install librdkafka
+ shell: "{{ item }}"
+ args:
+ chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
+ creates: "{{ librdkafka_home }}/lib/librdkafka.so"
+ with_items:
+ - rm -rf build/
+ - "./configure --prefix={{ librdkafka_home }} --enable-sasl"
+ - make
+ - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/librdkafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/librdkafka/tasks/main.yml b/metron-deployment/ansible/roles/librdkafka/tasks/main.yml
new file mode 100644
index 0000000..2144d7f
--- /dev/null
+++ b/metron-deployment/ansible/roles/librdkafka/tasks/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: dependencies.yml
+- include: librdkafka.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml b/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml
new file mode 100644
index 0000000..9969593
--- /dev/null
+++ b/metron-deployment/ansible/roles/libselinux-python/tasks/main.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install libselinux-python
+ yum:
+ name: libselinux-python
+ state: installed
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
+ when: ansible_distribution == "CentOS"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/load_web_templates/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/load_web_templates/meta/main.yml b/metron-deployment/ansible/roles/load_web_templates/meta/main.yml
new file mode 100644
index 0000000..c3d807b
--- /dev/null
+++ b/metron-deployment/ansible/roles/load_web_templates/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
new file mode 100644
index 0000000..3a91960
--- /dev/null
+++ b/metron-deployment/ansible/roles/load_web_templates/tasks/main.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Load Kibana Dashboard
+ command: >
+ curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests
+ args:
+ warn: off
+ register: result
+ failed_when: "result.rc != 0 or '202' not in result.stdout"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/defaults/main.yml b/metron-deployment/ansible/roles/metron-builder/defaults/main.yml
new file mode 100644
index 0000000..1bdd0ff
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-builder/defaults/main.yml
@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+metron_build_dir: "{{ playbook_dir }}/../../.."
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml b/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml
new file mode 100644
index 0000000..4949196
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-builder/tasks/build-debs.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Build Metron DEB Packages
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ metron_build_dir }}/metron-deployment"
+ with_items:
+ - mvn package -DskipTests -Pbuild-debs
+ become: false
+ run_once: true
+ delegate_to: localhost
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml b/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml
new file mode 100644
index 0000000..c362fc2
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-builder/tasks/build-rpms.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Build Metron RPM Packages
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ metron_build_dir }}/metron-deployment"
+ with_items:
+ - mvn package -DskipTests -Pbuild-rpms
+ become: false
+ run_once: true
+ delegate_to: localhost
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/build.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/build.yml b/metron-deployment/ansible/roles/metron-builder/tasks/build.yml
new file mode 100644
index 0000000..b63d1ef
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-builder/tasks/build.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Build Metron
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ metron_build_dir }}"
+ with_items:
+ - mvn package -DskipTests -T 2C -P HDP-2.5.0.0,mpack
+ become: false
+ run_once: true
+ delegate_to: localhost
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml b/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml
new file mode 100644
index 0000000..4497d82
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-builder/tasks/clean.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# clean out build artifacts including packages
+#
+- name: Clean Metron
+ shell: "{{ item }}"
+ args:
+ chdir: "{{ metron_build_dir }}"
+ with_items:
+ - mvn clean -P HDP-2.5.0.0
+ - mvn clean -P mpack
+ - mvn clean -P build-rpms
+ - mvn clean -P build-debs
+ become: false
+ run_once: true
+ delegate_to: localhost
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-builder/tasks/main.yml b/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
new file mode 100644
index 0000000..9636363
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-builder/tasks/main.yml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# fail if the distribution is unknown or unsupported
+#
+- name: Check distribution
+ fail:
+ msg: "Unsupported distribution; no packages available for {{ ansible_distribution }}"
+ when: (ansible_distribution != "Ubuntu") and (ansible_distribution != "CentOS")
+
+- include: clean.yml
+- include: build.yml
+
+- include: build-rpms.yml
+ static: no
+ when: ansible_distribution == "CentOS"
+
+- include: build-debs.yml
+ static: no
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/defaults/main.yml b/metron-deployment/ansible/roles/metron-packages/defaults/main.yml
new file mode 100644
index 0000000..36de438
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+repo_path: "/localrepo"
+metron_rpm_glob: "{{ playbook_dir }}/../../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm"
+metron_deb_glob: "{{ playbook_dir }}/../../packaging/docker/deb-docker/target/*.deb"
+packages_required: true
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml
new file mode 100644
index 0000000..cd76fa3
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_centos.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Copy Metron RPMs
+ copy:
+ src: "{{ item }}"
+ dest: "{{ repo_path }}"
+ owner: root
+ mode: 0755
+ with_fileglob:
+ - "{{ metron_rpm_glob }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml
new file mode 100644
index 0000000..6437bb5
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/copy_packages_ubuntu.yml
@@ -0,0 +1,31 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create local repo structure
+ file:
+ path: "{{ repo_path }}/dists/METRON/main/binary-amd64"
+ state: directory
+ mode: 0755
+
+- name: Copy Metron DEBs
+ copy:
+ src: "{{ item }}"
+ dest: "{{ repo_path }}/dists/METRON/main/binary-amd64"
+ owner: root
+ mode: 0755
+ with_fileglob:
+ - "{{ metron_deb_glob }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml b/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml
new file mode 100644
index 0000000..aae8bee
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/create_directory.yml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Clean out local repo
+ file:
+ state: absent
+ path: "{{ repo_path }}"
+
+- name: Create local repo directory
+ file:
+ path: "{{ repo_path }}"
+ state: directory
+ mode: 0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml
new file mode 100644
index 0000000..81c265d
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_centos.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install tooling on CentOS
+ package:
+ name: createrepo
+ state: installed
+
+- name: Create local repo with new packages on CentOS
+ shell: "createrepo {{ repo_path }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml
new file mode 100644
index 0000000..95a214f
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/create_repo_ubuntu.yml
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install tooling on Ubuntu
+ package:
+ name: dpkg-dev
+ state: installed
+
+- name: Create repository index directory on Ubuntu
+ file:
+ path: "{{ repo_path }}/dists/METRON/main/binary-amd64"
+ state: directory
+ mode: 0755
+
+- name: Create repository index on Ubuntu
+ shell: "dpkg-scanpackages . /dev/null | gzip -9c > dists/METRON/main/binary-amd64/Packages.gz"
+ args:
+ chdir: "{{ repo_path }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/main.yml b/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
new file mode 100644
index 0000000..193a64c
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/main.yml
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: validate.yml
+- include: create_directory.yml
+
+#
+# copy metron packages to the remote host
+#
+# the repository on the remote host containing the Metron packages (by default
+# located at /localrepo) MUST BE defined on every Ambari slave host. in most
+# cases, this local repository can be empty. only the local repository hosting
+# metron needs to contain the metron packages (either RPMs or DEBs)
+#
+# 'packages_required'
+#
+# this variable defines whether the Metron packages need to be copied to the
+# repository or whether it can be left empty. copying the packages can be a
+# time consuming operation on a large cluster, so it avoided unless required.
+#
+- include: copy_packages_centos.yml
+ static: no
+ when: ansible_distribution == "CentOS" and packages_required == true
+
+- include: copy_packages_ubuntu.yml
+ static: no
+ when: ansible_distribution == "Ubuntu" and packages_required == true
+
+#
+# create the local repository
+#
+- include: create_repo_centos.yml
+ static: no
+ when: ansible_distribution == "CentOS"
+
+- include: create_repo_ubuntu.yml
+ static: no
+ when: ansible_distribution == "Ubuntu"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml b/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml
new file mode 100644
index 0000000..8a23ab1
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron-packages/tasks/validate.yml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# fail if the distribution is unknown or unsupported
+#
+- name: Validate support for distribution
+ fail:
+ msg: "Unsupported distribution; no packages available for {{ ansible_distribution }}"
+ when: (ansible_distribution != "CentOS") and (ansible_distribution != "Ubuntu")
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template b/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template
new file mode 100644
index 0000000..3bb4633
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_elasticsearch_templates/files/es_templates/error_index.template
@@ -0,0 +1,57 @@
+{
+ "template": "error_index*",
+ "mappings": {
+ "error_doc": {
+ "_timestamp": {
+ "enabled": true
+ },
+ "properties": {
+ "exception": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "hostname": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "stack": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "timestamp": {
+ "type": "date",
+ "format": "epoch_millis"
+ },
+ "message": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "raw_message": {
+ "type": "string",
+ "index": "not_analyzed",
+ "ignore_above": 8191
+ },
+ "raw_message_bytes": {
+ "type": "binary",
+ "index": "no"
+ },
+ "error_fields": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "error_hash": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "failed_sensor_type": {
+ "type": "string",
+ "index": "not_analyzed"
+ },
+ "error_type": {
+ "type": "string",
+ "index": "not_analyzed"
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml
new file mode 100644
index 0000000..f570602
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/defaults/main.yml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+metron_version: 0.4.3
+metron_directory: /usr/metron/{{ metron_version }}
+pcapservice_jar_name: metron-api-{{ metron_version }}.jar
+pcapservice_jar_src: "{{ playbook_dir }}/../../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}"
+pcapservice_jar_dst: "{{ metron_directory }}/lib/{{ pcapservice_jar_name }}"
+pcapservice_port: 8081
+hbase_config_path: "/etc/hbase/conf"
+query_hdfs_path: "/tmp"
+pcap_hdfs_path: "/apps/metron/pcap"
+metron_pcapservice_logrotate_frequency: daily
+metron_pcapservice_logrotate_retention: 30
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml b/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml
new file mode 100644
index 0000000..ddf6aa9
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/meta/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+dependencies:
+ - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml b/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml
new file mode 100644
index 0000000..b77c1ec
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/tasks/config-hbase.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Fetch hbase-site.xml
+ fetch: src=/etc/hbase/conf/hbase-site.xml dest=/tmp/hbase/conf/hbase-site.xml flat=yes
+ delegate_to: "{{ groups.ambari_slave[0] }}"
+
+ - name: Create hbase conf directory
+ file: path=/etc/hbase/conf state=directory mode=0755
+
+ - name: Copy hbase-site.xml
+ copy: src=/tmp/hbase/conf/hbase-site.xml dest=/etc/hbase/conf/hbase-site.xml mode=0644
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml b/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml
new file mode 100644
index 0000000..ed33354
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/tasks/main.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- include: pcapservice.yml
+
+- name: Create Logrotate Script for metron_pcapservice
+ template:
+ src: "metron-pcapservice-logrotate.yml"
+ dest: "/etc/logrotate.d/metron-pcapservice"
+ mode: 0644
+
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml b/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml
new file mode 100644
index 0000000..d0ac411
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/tasks/pcapservice.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create Metron streaming directories
+ file: path={{ metron_directory }}/{{ item.name }} state=directory mode=0755
+ with_items:
+ - { name: 'lib'}
+ - { name: 'config'}
+
+- name: Copy Metron pcapservice jar
+ copy:
+ src: "{{ pcapservice_jar_src }}"
+ dest: "{{ pcapservice_jar_dst }}"
+
+- name: Install service script
+ template: src=pcapservice dest=/etc/init.d/pcapservice mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml b/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
new file mode 100644
index 0000000..0293ade
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Metron pcapService
+/var/log/metron_pcapservice.log {
+ {{ metron_pcapservice_logrotate_frequency }}
+ rotate {{ metron_pcapservice_logrotate_retention }}
+ missingok
+ notifempty
+ copytruncate
+ compress
+}
+
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice b/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice
new file mode 100644
index 0000000..054133d
--- /dev/null
+++ b/metron-deployment/ansible/roles/metron_pcapservice/templates/pcapservice
@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# metron pcap service
+# chkconfig: 345 20 80
+# description: Metron PCAP Service Daemon
+# processname: pcapservice
+#
+NAME=pcapservice
+DESC="Metron pcap service"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+LOGFILE="/var/log/metron_pcapservice.log"
+EXTRA_ARGS="${@:2}"
+DAEMON_PATH="/"
+DAEMON="/usr/bin/yarn jar"
+DAEMONOPTS="{{ pcapservice_jar_dst }} org.apache.metron.pcapservice.rest.PcapService -port {{ pcapservice_port }} -query_hdfs_path {{ query_hdfs_path }} -pcap_hdfs_path {{ pcap_hdfs_path }}"
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ kill -HUP $PID
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
[33/50] [abbrv] metron git commit: METRON-1427: Add support for storm
1.1 and hdp 2.6 (cstella via mmiklavc) closes apache/metron#907
Posted by rm...@apache.org.
METRON-1427: Add support for storm 1.1 and hdp 2.6 (cstella via mmiklavc) closes apache/metron#907
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/644e951c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/644e951c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/644e951c
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 644e951c8b2c33b5e52602a814d91012c3b325b1
Parents: 0874571
Author: cstella <ce...@gmail.com>
Authored: Tue Jan 30 10:54:57 2018 -0700
Committer: Michael Miklavcic <mi...@gmail.com>
Committed: Tue Jan 30 10:54:57 2018 -0700
----------------------------------------------------------------------
metron-analytics/metron-profiler/pom.xml | 4 +-
.../roles/ambari_common/defaults/main.yml | 4 +-
.../roles/ambari_config/defaults/main.yml | 2 +-
.../roles/ambari_config/vars/single_node_vm.yml | 7 ++
.../roles/ambari_config/vars/small_cluster.yml | 7 ++
.../roles/ambari_gather_facts/defaults/main.yml | 19 ++++++
.../roles/ambari_gather_facts/tasks/main.yml | 67 ++++++++++++--------
.../metron-mpack/src/main/resources/mpack.json | 14 ++++
.../apache/metron/rest/config/KafkaConfig.java | 8 ++-
.../rest/service/impl/StormCLIWrapper.java | 3 +-
.../apache/metron/common/utils/KafkaUtils.java | 27 ++++++++
metron-platform/metron-elasticsearch/pom.xml | 4 +-
.../parsers/topology/ParserTopologyBuilder.java | 3 +-
metron-platform/metron-solr/pom.xml | 12 +++-
.../kafka/flux/SimpleStormKafkaBuilder.java | 2 +
.../apache/metron/writer/kafka/KafkaWriter.java | 1 +
16 files changed, 144 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-analytics/metron-profiler/pom.xml
----------------------------------------------------------------------
diff --git a/metron-analytics/metron-profiler/pom.xml b/metron-analytics/metron-profiler/pom.xml
index d634cef..4d36782 100644
--- a/metron-analytics/metron-profiler/pom.xml
+++ b/metron-analytics/metron-profiler/pom.xml
@@ -341,8 +341,8 @@
<shadedPattern>org.apache.metron.guava.metron-profiler</shadedPattern>
</relocation>
<relocation>
- <pattern>com.fasterxml.jackson.core</pattern>
- <shadedPattern>com.fasterxml.jackson.core.metron.elasticsearch</shadedPattern>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
</relocation>
</relocations>
<artifactSet>
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
index c04400e..0614e0f 100644
--- a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
@@ -17,7 +17,7 @@
---
hadoop_logrotate_frequency: daily
hadoop_logrotate_retention: 30
-centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.4.2.0/ambari.repo
-ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.4.2.0
+centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.5.2.0/ambari.repo
+ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.5.2.0
ubuntu_elasticsearch_packages_repo: https://artifacts.elastic.co/packages/5.x/apt
ubuntu_elasticsearch_curator_repo: https://packages.elastic.co/curator/5/debian
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
index e0de145..ad7ca9e 100644
--- a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
@@ -34,5 +34,5 @@ mapred_reduce_java_opts : -Xmx1024m
mapred_map_mem_mb : 1229
mapred_reduce_mem_mb : 1229
topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf'
-hdp_stack: "2.5"
+hdp_stack: "2.6"
elasticsearch_network_interface: _site_
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
index 6a60902..bf54fe0 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/single_node_vm.yml
@@ -87,6 +87,13 @@ configurations:
supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
storm.local.dir: '{{ storm_local_dir }}'
topology.classpath: '{{ topology_classpath }}'
+ # Storm expects ambari metrics to be available in 2.6. We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer
+ storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]'
+ topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
+ # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties
+ nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
+ supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
+ worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
- kafka-env:
content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
- kafka-broker:
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
index 4ec8458..218e267 100644
--- a/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
+++ b/metron-deployment/ansible/roles/ambari_config/vars/small_cluster.yml
@@ -85,6 +85,13 @@ configurations:
supervisor.slots.ports: "[6700, 6701, 6702, 6703, 6704, 6705]"
storm.local.dir: '{{ storm_local_dir | default("/hadoop/storm") }}'
topology.classpath: '{{ topology_classpath }}'
+ # Storm expects ambari metrics to be available in 2.6. We do *not* install ambari metrics in full-dev, so we need to revert to the old consumer
+ storm.cluster.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer"}]'
+ topology.metrics.consumer.register: '[{"class": "org.apache.storm.metric.LoggingMetricsConsumer", "parallelism.hint": 1, "whitelist": ["kafkaOffset\\..+/", "__complete-latency", "__process-latency", "__receive\\.population$", "__sendqueue\\.population$", "__execute-count", "__emit-count", "__ack-count", "__fail-count", "memory/heap\\.usedBytes$", "memory/nonHeap\\.usedBytes$", "GC/.+\\.count$", "GC/.+\\.timeMs$"]}]'
+ # Storm expects ambari metrics to be available in 2.6 and ambari metrics pulls data via JMX, but since we don't use ambari metrics here, we don't have the javaagent around to use and thus that must be removed from nimbus, supervisor and worker properties
+ nimbus.childopts: '-Xmx1024m _JAAS_PLACEHOLDER'
+ supervisor.childopts: '-Xmx256m _JAAS_PLACEHOLDER'
+ worker.childopts: "-Xmx768m _JAAS_PLACEHOLDER"
- kafka-broker:
log.dirs: '{{ kafka_log_dirs | default("/kafka-log") }}'
- metron-rest-env:
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml
new file mode 100644
index 0000000..5351a60
--- /dev/null
+++ b/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+curl: "curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H \"X-Requested-By: ambari\""
+parse_json: "import sys, json; print json.load(sys.stdin)"
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
index 2b37eec..25f0982 100644
--- a/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_gather_facts/tasks/main.yml
@@ -32,55 +32,55 @@
cluster_name: "{{ (cluster_name_response.content | from_json)['items'][0].Clusters.cluster_name }}"
when: cluster_name is undefined
+- set_fact:
+ base_url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}"
+
#
# namenode_host
#
- name: "Ask Ambari: namenode_host"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: namenode_hosts_response
+ shell: >
+ {{ curl }} '{{ base_url }}/services/HDFS/components/NAMENODE' \
+ | python -c '{{ parse_json }}["host_components"][0]["HostRoles"]["host_name"]'
+ args:
+ warn: false
+ register: namenode_host_response
when: namenode_host is undefined
- set_fact:
- namenode_host: "{{ (namenode_hosts_response.content | from_json).host_components[0].HostRoles.host_name }}"
+ namenode_host: "{{ namenode_host_response.stdout_lines[0] }}"
when: namenode_host is undefined
#
# core_site_tag
#
- name: "Ask Ambari: core_site_tag"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
+ shell: >
+ {{ curl }} '{{ base_url }}/hosts/{{ namenode_host }}/host_components/NAMENODE' \
+ | python -c '{{ parse_json }}["HostRoles"]["actual_configs"]["core-site"]["default"]'
+ args:
+ warn: false
register: core_site_tag_response
when: core_site_tag is undefined
- set_fact:
- core_site_tag: "{{ (core_site_tag_response.content | from_json).HostRoles.actual_configs['core-site'].default }}"
+ core_site_tag: "{{ core_site_tag_response.stdout_lines[0] }}"
when: core_site_tag is undefined
#
# hdfs_url
#
- name: "Ask Ambari: hdfs_url"
- uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}"
- user: "{{ ambari_user }}"
- password: "{{ ambari_password }}"
- force_basic_auth: yes
- return_content: yes
- register: core_site_response
+ shell: >
+ {{ curl }} '{{ base_url }}/configurations?type=core-site&tag={{ core_site_tag }}' \
+ | python -c '{{ parse_json }}["items"][0]["properties"]["fs.defaultFS"]'
+ args:
+ warn: false
+ register: hdfs_url_response
when: hdfs_url is undefined
- set_fact:
- hdfs_url: "{{ (core_site_response.content | from_json)['items'][0].properties['fs.defaultFS'] }}"
+ hdfs_url: "{{ hdfs_url_response.stdout_lines[0] }}"
when: hdfs_url is undefined
#
@@ -88,7 +88,7 @@
#
- name: "Ask Ambari: kafka_broker_hosts"
uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER"
+ url: "{{ base_url }}/services/KAFKA/components/KAFKA_BROKER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -105,7 +105,7 @@
#
- name: "Ask Ambari: kafka_broker_tag"
uri:
- url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
+ url: "{{ base_url }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -122,7 +122,8 @@
#
- name: "Ask Ambari: kafka_broker_port"
shell: >
- curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H "X-Requested-By: ambari" "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}" | python -c 'import sys, json; print json.load(sys.stdin)["items"][0]["properties"]["listeners"]'
+ {{ curl }} '{{ base_url }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}' \
+ | python -c '{{ parse_json }}["items"][0]["properties"]["listeners"]'
args:
warn: false
register: kafka_broker_port_response
@@ -191,6 +192,9 @@
zookeeper_url: "{% for host in zookeeper_hosts %}{% if loop.index != 1 %},{% endif %}{{ host }}:{{ zookeeper_port }}{% endfor %}"
when: zookeeper_url is undefined
+#
+# metron_hosts
+#
- name: "Ask Ambari: metron_hosts"
uri:
url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/METRON/components/METRON_INDEXING"
@@ -205,6 +209,9 @@
metron_hosts: "{{ (metron_hosts_response.content | from_json).host_components | map(attribute='HostRoles.host_name') | list }}"
when: metron_hosts is undefined
+#
+# kibana hosts
+#
- name: "Ask Ambari: kibana_hosts"
uri:
url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KIBANA/components/KIBANA_MASTER"
@@ -225,10 +232,14 @@
#
- name: debug
debug:
- msg: "zookeeper_port = {{ zookeeper_port }},
+ msg: "cluster_name = {{ cluster_name }},
+ namenode_host = {{ namenode_host }},
+ hdfs_url = {{ hdfs_url }},
+ zookeeper_port = {{ zookeeper_port }},
zookeeper_hosts = {{ zookeeper_hosts }},
zookeeper_url = {{ zookeeper_url }},
kafka_broker_port = {{ kafka_broker_port }},
kafka_broker_hosts = {{ kafka_broker_hosts }},
kafka_broker_url = {{ kafka_broker_url }},
- metron_hosts = {{ metron_hosts }}"
+ metron_hosts = {{ metron_hosts }},
+ kibana_hosts = {{ kibana_hosts }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
index 7a9d892..3946881 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
+++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/mpack.json
@@ -38,7 +38,12 @@
{
"stack_name" : "HDP",
"stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
}
+
]
},
{
@@ -56,8 +61,13 @@
{
"stack_name" : "HDP",
"stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
}
+
]
},
{
@@ -75,6 +85,10 @@
{
"stack_name" : "HDP",
"stack_version" : "2.5"
+ },
+ {
+ "stack_name" : "HDP",
+ "stack_version" : "2.6"
}
]
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
index a15c48f..7e9b468 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/KafkaConfig.java
@@ -22,6 +22,8 @@ import kafka.utils.ZkUtils;
import org.I0Itec.zkclient.ZkClient;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.producer.KafkaProducer;
+import org.apache.kafka.common.protocol.SecurityProtocol;
+import org.apache.metron.common.utils.KafkaUtils;
import org.apache.metron.rest.MetronRestConstants;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
@@ -86,7 +88,7 @@ public class KafkaConfig {
props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");
if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) {
- props.put("security.protocol", environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY));
+ props.put("security.protocol", KafkaUtils.INSTANCE.normalizeProtocol(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY)));
}
return props;
}
@@ -109,11 +111,13 @@ public class KafkaConfig {
producerConfig.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer");
producerConfig.put("request.required.acks", 1);
if (environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false)) {
- producerConfig.put("security.protocol", environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY));
+ producerConfig.put("security.protocol", KafkaUtils.INSTANCE.normalizeProtocol(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY)));
}
return producerConfig;
}
+
+
@Bean
public KafkaProducer kafkaProducer() {
return new KafkaProducer<>(producerProperties());
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
index 463c925..fff7390 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/StormCLIWrapper.java
@@ -18,6 +18,7 @@
package org.apache.metron.rest.service.impl;
import org.apache.commons.lang3.StringUtils;
+import org.apache.metron.common.utils.KafkaUtils;
import org.apache.metron.rest.MetronRestConstants;
import org.apache.metron.rest.RestException;
import org.slf4j.Logger;
@@ -117,7 +118,7 @@ public class StormCLIWrapper {
// kafka security protocol
command.add( "-ksp");
- command.add( environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY));
+ command.add(KafkaUtils.INSTANCE.normalizeProtocol(environment.getProperty(MetronRestConstants.KAFKA_SECURITY_PROTOCOL_SPRING_PROPERTY)));
// extra topology options
boolean kerberosEnabled = environment.getProperty(MetronRestConstants.KERBEROS_ENABLED_SPRING_PROPERTY, Boolean.class, false);
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
index d54e2b8..796bc42 100644
--- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
+++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/utils/KafkaUtils.java
@@ -25,6 +25,7 @@ import org.apache.curator.RetryPolicy;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.kafka.common.protocol.SecurityProtocol;
import java.util.ArrayList;
import java.util.List;
@@ -32,6 +33,7 @@ import java.util.Map;
public enum KafkaUtils {
INSTANCE;
+ public static final String SECURITY_PROTOCOL = "security.protocol";
public List<String> getBrokersFromZookeeper(String zkQuorum) throws Exception {
RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
CuratorFramework framework = CuratorFrameworkFactory.newClient(zkQuorum, retryPolicy);
@@ -66,6 +68,31 @@ public enum KafkaUtils {
return ret;
}
+ public Map<String, Object> normalizeProtocol(Map<String, Object> configs) {
+ if(configs.containsKey(SECURITY_PROTOCOL)) {
+ String protocol = normalizeProtocol((String)configs.get(SECURITY_PROTOCOL));
+ configs.put(SECURITY_PROTOCOL, protocol);
+ }
+ return configs;
+ }
+
+ public String normalizeProtocol(String protocol) {
+ if(protocol.equalsIgnoreCase("PLAINTEXTSASL") || protocol.equalsIgnoreCase("SASL_PLAINTEXT")) {
+ if(SecurityProtocol.getNames().contains("PLAINTEXTSASL")) {
+ return "PLAINTEXTSASL";
+ }
+ else if(SecurityProtocol.getNames().contains("SASL_PLAINTEXT")) {
+ return "SASL_PLAINTEXT";
+ }
+ else {
+ throw new IllegalStateException("Unable to find the appropriate SASL protocol, " +
+ "viable options are: " + Joiner.on(",").join(SecurityProtocol.getNames()));
+ }
+ }
+ else {
+ return protocol.trim();
+ }
+ }
/*
The URL accepted is NOT a general URL, and is assumed to follow the format used by the Kafka structures in Zookeeper.
See: https://cwiki.apache.org/confluence/display/KAFKA/Kafka+data+structures+in+Zookeeper
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-platform/metron-elasticsearch/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml
index 97f4062..141d8aa 100644
--- a/metron-platform/metron-elasticsearch/pom.xml
+++ b/metron-platform/metron-elasticsearch/pom.xml
@@ -264,8 +264,8 @@
<shadedPattern>org.apache.metron.guava.metron-elasticsearch</shadedPattern>
</relocation>
<relocation>
- <pattern>com.fasterxml.jackson.core</pattern>
- <shadedPattern>com.fasterxml.jackson.core.metron.elasticsearch</shadedPattern>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
</relocation>
</relocations>
<artifactSet>
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
index c918703..1039e56 100644
--- a/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
+++ b/metron-platform/metron-parsers/src/main/java/org/apache/metron/parsers/topology/ParserTopologyBuilder.java
@@ -18,6 +18,7 @@
package org.apache.metron.parsers.topology;
import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.apache.metron.common.utils.KafkaUtils;
import org.apache.metron.parsers.topology.config.ValueSupplier;
import org.apache.metron.storm.kafka.flux.SimpleStormKafkaBuilder;
import org.apache.metron.storm.kafka.flux.SpoutConfiguration;
@@ -163,7 +164,7 @@ public class ParserTopologyBuilder {
, inputTopic + "_parser"
);
if(securityProtocol.isPresent()) {
- kafkaSpoutConfigOptions.putIfAbsent("security.protocol", securityProtocol.get());
+ kafkaSpoutConfigOptions.putIfAbsent("security.protocol", KafkaUtils.INSTANCE.normalizeProtocol(securityProtocol.get()));
}
return SimpleStormKafkaBuilder.create( inputTopic
, zkQuorum
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-platform/metron-solr/pom.xml
----------------------------------------------------------------------
diff --git a/metron-platform/metron-solr/pom.xml b/metron-platform/metron-solr/pom.xml
index a2eee71..9c9c7fb 100644
--- a/metron-platform/metron-solr/pom.xml
+++ b/metron-platform/metron-solr/pom.xml
@@ -261,7 +261,17 @@
<exclude>META-INF/*.RSA</exclude>
</excludes>
</filter>
- </filters>
+ </filters>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.apache.metron.guava</shadedPattern>
+ </relocation>
+ <relocation>
+ <pattern>com.fasterxml.jackson</pattern>
+ <shadedPattern>org.apache.metron.jackson</shadedPattern>
+ </relocation>
+ </relocations>
<artifactSet>
<excludes>
<exclude>storm:storm-core:*</exclude>
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java b/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
index 1bcee9a..f99e549 100644
--- a/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
+++ b/metron-platform/metron-storm-kafka/src/main/java/org/apache/metron/storm/kafka/flux/SimpleStormKafkaBuilder.java
@@ -203,6 +203,8 @@ public class SimpleStormKafkaBuilder<K, V> extends KafkaSpoutConfig.Builder<K, V
, createDeserializer(Optional.ofNullable((String)kafkaProps.get(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG)), DEFAULT_DESERIALIZER)
, subscription
);
+
+ kafkaProps = KafkaUtils.INSTANCE.normalizeProtocol(kafkaProps);
setProp(kafkaProps);
setRecordTranslator(new SpoutRecordTranslator<>(FieldsConfiguration.toList(fieldsConfiguration)));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/644e951c/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
----------------------------------------------------------------------
diff --git a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
index 7ce9b9b..f73e0f4 100644
--- a/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
+++ b/metron-platform/metron-writer/src/main/java/org/apache/metron/writer/kafka/KafkaWriter.java
@@ -158,6 +158,7 @@ public class KafkaWriter extends AbstractWriter implements MessageWriter<JSONObj
producerConfig.put("value.serializer", valueSerializer);
producerConfig.put("request.required.acks", requiredAcks);
producerConfig.putAll(producerConfigs == null?new HashMap<>():producerConfigs);
+ producerConfig = KafkaUtils.INSTANCE.normalizeProtocol(producerConfig);
return producerConfig;
}
[38/50] [abbrv] metron git commit: METRON-1443 Missing Critical MPack
Install Instruction for Ubuntu (nickwallen) closes apache/metron#925
Posted by rm...@apache.org.
METRON-1443 Missing Critical MPack Install Instruction for Ubuntu (nickwallen) closes apache/metron#925
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/06305052
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/06305052
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/06305052
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 06305052a186ff911d204c755804e7225d05bf28
Parents: c26abbb
Author: nickwallen <ni...@nickallen.org>
Authored: Mon Feb 5 09:17:57 2018 -0500
Committer: nickallen <ni...@apache.org>
Committed: Mon Feb 5 09:17:57 2018 -0500
----------------------------------------------------------------------
.../packaging/ambari/metron-mpack/README.md | 14 ++++++++++++++
1 file changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/06305052/metron-deployment/packaging/ambari/metron-mpack/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/packaging/ambari/metron-mpack/README.md b/metron-deployment/packaging/ambari/metron-mpack/README.md
index 5179b5f..4b0b073 100644
--- a/metron-deployment/packaging/ambari/metron-mpack/README.md
+++ b/metron-deployment/packaging/ambari/metron-mpack/README.md
@@ -28,6 +28,20 @@ This allows you to easily install Metron using a simple, guided process. This a
* A [Node.js](https://nodejs.org/en/download/package-manager/) repository installed on the host running the Management and Alarm UI.
+* When installing on Ubuntu the Elasticsearch repository must be defined manually. This is NOT defined by the Mpack like it is on CentOS. This is an open bug that needs addressed in the Mpack. See the [Elasticsearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html) documentation for more specific instructions.
+ ```
+ $ cat >/etc/apt/sources.list.d/elasticsearch.list << EOL
+ deb https://packages.elastic.co/curator/5/debian stable main
+ deb https://artifacts.elastic.co/packages/5.x/apt stable main
+ EOL
+
+ $ wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+
+ $ apt-get update
+ ```
+
+
+
### Quick Start
1. Build the Metron MPack. Execute the following command from the project's root directory.
[10/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/tasks/main.yml b/metron-deployment/roles/sensor-stubs/tasks/main.yml
deleted file mode 100644
index 708880f..0000000
--- a/metron-deployment/roles/sensor-stubs/tasks/main.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create home directory
- file: path={{ item }} state=directory mode=0755
- with_items:
- - "{{ sensor_stubs_home }}"
- - "{{ sensor_stubs_bin }}"
- - "{{ sensor_stubs_data }}"
-
-- name: Distribute stub data
- copy: src={{ item }} dest={{ sensor_stubs_data }}/
- with_items:
- - bro.out
- - snort.out
- - yaf.out
-
-- name: Install service script
- template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755
-
-- name: Install sensor stubs
- template: src={{ item }} dest={{ sensor_stubs_bin }}/ mode=0755
- with_items:
- - start-bro-stub
- - start-snort-stub
- - start-yaf-stub
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/templates/sensor-stubs
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/sensor-stubs b/metron-deployment/roles/sensor-stubs/templates/sensor-stubs
deleted file mode 100644
index eaab69d..0000000
--- a/metron-deployment/roles/sensor-stubs/templates/sensor-stubs
+++ /dev/null
@@ -1,154 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# metron sensor-stubs service
-# chkconfig: 345 20 80
-# description: Simulates the behavior of a sensor by sending canned telemetry data to a Kafka topic
-# processname: sensor-stubs
-#
-NAME=sensor-stubs
-DESC="Simulates the behavior of a sensor by sending canned telemetry data to a Kafka topic"
-SCRIPTNAME=/etc/init.d/$NAME
-LOGFILE="{{ sensor_stubs_log }}"
-DAEMON_PATH="{{ sensor_stubs_home }}"
-DATA="{{ sensor_stubs_data }}"
-
-# the delay between each 'batch' of messages in seconds.
-# see {{ sensor_stubs_bin }}/start-sensor-stub for more information.
-DELAY={{ sensor_stubs_delay }}
-
-# the number of messages to send in each batch.
-# see {{ sensor_stubs_bin }}/start-sensor-stub for more information.
-COUNT={{ sensor_stubs_count }}
-
-#
-# which sensors? defaults to bro, snort, yaf
-#
-SENSORS="${@:2}"
-if [ -z "${SENSORS}" ]; then
- SENSORS=('bro' 'yaf' 'snort')
-fi
-
-#
-# start a sensor stub
-#
-start() {
-
- # if pidfile exists, do not start another
- PIDFILE="/var/run/$NAME-$1.pid"
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- printf "%10s: %s \n" "$1" "OK [$PID]"
- return
- fi
-
- # kick-off the daemon
- DAEMON="{{ sensor_stubs_bin }}/start-$1-stub $DELAY $COUNT"
- PID=`$DAEMON >> $LOGFILE 2>&1 & echo $!`
-
- if [ -z $PID ]; then
- printf "%10s: %s \n" "$1" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%10s: %s \n" "$1" "Ok [$PID]"
- fi
-}
-
-#
-# stop a sensor stub
-#
-stop() {
- PIDFILE="/var/run/$NAME-$1.pid"
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- while sleep 1
- echo -n "."
- kill -0 $PID >/dev/null 2>&1
- do
- kill $PID
- done
-
- printf "%10s: %s \n" "$1" "Stopped [$PID]"
- rm -f $PIDFILE
- else
- printf "%10s: %s \n" "$1" "Not running"
- fi
-}
-
-#
-# status check of sensor stub
-#
-status() {
- PIDFILE="/var/run/$NAME-$1.pid"
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%10s: %s \n" "$1" "Process dead but pidfile exists"
- else
- printf "%10s: %s \n" "$1" "Running [${PID}]"
- fi
- else
- printf "%10s: %s \n" "$1" "Not running"
- fi
-}
-
-case "$1" in
-
- ##############################################################################
- # start
- #
- start)
- printf "%-50s \n" "Starting $NAME..."
- for sensor in "${SENSORS[@]}"; do
- start $sensor
- done
- ;;
-
- ##############################################################################
- # status
- #
- status)
- printf "%-50s \n" "Checking $NAME..."
- for sensor in "${SENSORS[@]}"; do
- status $sensor
- done
- ;;
-
- ##############################################################################
- # stop
- #
- stop)
- printf "%-50s \n" "Stopping $NAME..."
- for sensor in "${SENSORS[@]}"; do
- stop $sensor
- done
- ;;
-
- ##############################################################################
- # restart
- #
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/templates/start-bro-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/start-bro-stub b/metron-deployment/roles/sensor-stubs/templates/start-bro-stub
deleted file mode 100644
index 979de3d..0000000
--- a/metron-deployment/roles/sensor-stubs/templates/start-bro-stub
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# simulates the behavior of a sensor by sending canned telemetry data
-# to a Kafka topic.
-#
-# a subset of the canned data is randomly selected and is sent in
-# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
-# the time delay between batches can be configured.
-#
-# start-bro-stub <DELAY> <COUNT>
-#
-
-#
-# how long to delay between each 'batch' in seconds.
-#
-DELAY=${1:-{{ sensor_stubs_delay }}}
-
-#
-# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
-#
-COUNT=${2:-{{ sensor_stubs_count }}}
-
-INPUT="{{ sensor_stubs_data }}/bro.out"
-PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
-TOPIC="bro"
-
-while true; do
-
- # transform the bro timestamp and push to kafka
- SEARCH="\"ts\"\:[0-9]\+\."
- REPLACE="\"ts\"\:`date +%s`\."
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC
-
- sleep $DELAY
-done
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/templates/start-snort-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/start-snort-stub b/metron-deployment/roles/sensor-stubs/templates/start-snort-stub
deleted file mode 100644
index 3123782..0000000
--- a/metron-deployment/roles/sensor-stubs/templates/start-snort-stub
+++ /dev/null
@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# simulates the behavior of a sensor by sending canned telemetry data
-# to a Kafka topic.
-#
-# a subset of the canned data is randomly selected and is sent in
-# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
-# the time delay between batches can be configured.
-#
-# start-snort-stub <DELAY> <COUNT>
-#
-
-#
-# how long to delay between each 'batch' in seconds.
-#
-DELAY=${1:-{{ sensor_stubs_delay }}}
-
-#
-# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
-#
-COUNT=${2:-{{ sensor_stubs_count }}}
-
-INPUT="{{ sensor_stubs_data }}/snort.out"
-PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
-TOPIC="snort"
-
-while true; do
-
- # transform the timestamp and push to kafka
- SEARCH="[^,]\+ ,"
- REPLACE="`date +'%m\/%d\/%y-%H:%M:%S'`.000000 ,"
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC
-
- sleep $DELAY
-done
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub b/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub
deleted file mode 100644
index 1966d39..0000000
--- a/metron-deployment/roles/sensor-stubs/templates/start-yaf-stub
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# simulates the behavior of a sensor by sending canned telemetry data
-# to a Kafka topic.
-#
-# a subset of the canned data is randomly selected and is sent in
-# batches. the timestamp of the message is altered to match current
-# system time. the number of messages sent in each batch, along with
-# the time delay between batches can be configured.
-#
-# start-yaf-stub <DELAY> <COUNT>
-#
-
-#
-# how long to delay between each 'batch' in seconds.
-#
-DELAY=${1:-{{ sensor_stubs_delay }}}
-
-#
-# how many messages to send in each 'batch'. the messages are drawn randomly
-# from the entire set of canned data.
-#
-COUNT=${2:-{{ sensor_stubs_count }}}
-
-
-INPUT="{{ sensor_stubs_data }}/yaf.out"
-PRODUCER="{{ kafka_home }}/bin/kafka-console-producer.sh"
-TOPIC="yaf"
-
-while true; do
-
- # transform the timestamp and push to kafka
- SEARCH="[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\} [0-9]\{2\}:[0-9]\{2\}:[0-9]\{2\}\.[0-9]\+"
- REPLACE="`date +'%Y-%m-%d %H:%M:%S'`.000"
- shuf -n $COUNT $INPUT | sed -e "s/$SEARCH/$REPLACE/g" | $PRODUCER --broker-list {{ kafka_broker_url }} --topic $TOPIC
-
- sleep $DELAY
-done
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/README.md b/metron-deployment/roles/sensor-test-mode/README.md
deleted file mode 100644
index 37afad8..0000000
--- a/metron-deployment/roles/sensor-test-mode/README.md
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-Sensor Test Mode
-================
-
-A role that configures each of the sensors to produce the maximum amount of telemetry data. This role is useful only for testing. It can be useful to support functional, performance, and load testing of Apache Metron.
-
-The role does the following to maximize the amount of telemetry data produced by each Metron sensor.
-
-- Plays a packet capture file through a network interface to simulate live network traffic.
-- Configures [YAF](https://tools.netsa.cert.org/yaf/yaf.html) with `idle-timeout=0`. This causes a flow record to be produced for every network packet received.
-- Configures [Snort](https://www.snort.org/) to produce an alert for every network packet received.
-
-Getting Started
----------------
-
-To enable the `sensor-test-mode` role apply the role to the `sensors` host group in your Ansible playbook.
-
-```
-- hosts: sensors
- roles:
- - role: sensor-test-mode
-```
-
-The role has also been added to the default `metron_install.yml` playbook so that it can be turned on/off with a property in both the local Virtualbox and the remote EC2 deployments.
-
-```
-sensor_test_mode: True
-```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/defaults/main.yml b/metron-deployment/roles/sensor-test-mode/defaults/main.yml
deleted file mode 100644
index 46c9750..0000000
--- a/metron-deployment/roles/sensor-test-mode/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-pcap_replay: True
-install_yaf: True
-install_snort: True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/files/example.pcap
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/files/example.pcap b/metron-deployment/roles/sensor-test-mode/files/example.pcap
deleted file mode 100644
index 06594ec..0000000
Binary files a/metron-deployment/roles/sensor-test-mode/files/example.pcap and /dev/null differ
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/meta/main.yml b/metron-deployment/roles/sensor-test-mode/meta/main.yml
deleted file mode 100644
index 0e9e5b3..0000000
--- a/metron-deployment/roles/sensor-test-mode/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - pcap_replay
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/main.yml b/metron-deployment/roles/sensor-test-mode/tasks/main.yml
deleted file mode 100644
index 24ca87e..0000000
--- a/metron-deployment/roles/sensor-test-mode/tasks/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: pcap.yml
- when: install_pcap_replay
-
-- include: yaf.yml
- when: install_yaf
-
-- include: snort.yml
- when: install_snort
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml b/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
deleted file mode 100644
index dda1bae..0000000
--- a/metron-deployment/roles/sensor-test-mode/tasks/pcap.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# load example pcap data to replay
-#
-- name: Install example pcap file
- copy: src=example.pcap dest={{ pcap_replay_home }}/
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/snort.yml b/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
deleted file mode 100644
index 3b1af5f..0000000
--- a/metron-deployment/roles/sensor-test-mode/tasks/snort.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# configure snort to alert on every packet
-#
-- name: Configure snort to use a set of test rules
- lineinfile:
- dest: /etc/snort/snort.conf
- line: "include $RULE_PATH/test.rules"
-
-- name: Create a snort alert for testing that alerts on every packet
- lineinfile:
- dest: /etc/snort/rules/test.rules
- line: "alert tcp any any -> any any (msg:'snort test alert'; sid:999158; )"
- create: yes
-
-- name: Configure home network
- lineinfile:
- dest: /etc/snort/snort.conf
- regexp: "^ipvar HOME_NET.*$"
- line: "ipvar HOME_NET any"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml b/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
deleted file mode 100644
index 64354ac..0000000
--- a/metron-deployment/roles/sensor-test-mode/tasks/yaf.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# configure yaf to generate a flow record for every packet
-#
-- name: Stop running instances of yaf
- service: name=yaf state=stopped
- ignore_errors: yes
-
-- name: Configure yaf to generate a flow record for every network packet
- lineinfile:
- dest: /etc/init.d/yaf
- regexp: "^DAEMONOPTS=\"${@:2}\"$"
- line: "DAEMONOPTS=\"${@:2} --idle-timeout 0\""
- backup: yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/defaults/main.yml b/metron-deployment/roles/snort/defaults/main.yml
deleted file mode 100644
index c8b126b..0000000
--- a/metron-deployment/roles/snort/defaults/main.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-snort_version: 2.9.8.0-1
-daq_version: 2.0.6-1
-snort_topic: snort
-snort_alert_csv_path: /var/log/snort/alert.csv
-snort_src_url: "https://snort.org/downloads/archive/snort/snort-{{ snort_version }}.src.rpm"
-snort_community_rules_url: "https://www.snort.org/downloads/community/community-rules.tar.gz"
-dag_src_url: "https://snort.org/downloads/snort/daq-{{ daq_version }}.src.rpm"
-sniff_interface: eth0
-snort_home_net: any
-snort_producer_home: /opt/snort-producer
-snort_producer_start: /opt/snort-producer/start-snort-producer.sh
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/files/snort.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/files/snort.conf b/metron-deployment/roles/snort/files/snort.conf
deleted file mode 100644
index b03247a..0000000
--- a/metron-deployment/roles/snort/files/snort.conf
+++ /dev/null
@@ -1,730 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-###################################################
-# This file contains a sample snort configuration.
-# You should take the following steps to create your own custom configuration:
-#
-# 1) Set the network variables.
-# 2) Configure the decoder
-# 3) Configure the base detection engine
-# 4) Configure dynamic loaded libraries
-# 5) Configure preprocessors
-# 6) Configure output plugins
-# 7) Customize your rule set
-# 8) Customize preprocessor and decoder rule set
-# 9) Customize shared object rule set
-###################################################
-
-###################################################
-# Step #1: Set the network variables. For more information, see README.variables
-###################################################
-
-# Setup the network addresses you are protecting
-ipvar HOME_NET 10.0.0.16
-
-# Set up the external network addresses. Leave as "any" in most situations
-ipvar EXTERNAL_NET any
-
-# List of DNS servers on your network
-ipvar DNS_SERVERS $HOME_NET
-
-# List of SMTP servers on your network
-ipvar SMTP_SERVERS $HOME_NET
-
-# List of web servers on your network
-ipvar HTTP_SERVERS $HOME_NET
-
-# List of sql servers on your network
-ipvar SQL_SERVERS $HOME_NET
-
-# List of telnet servers on your network
-ipvar TELNET_SERVERS $HOME_NET
-
-# List of ssh servers on your network
-ipvar SSH_SERVERS $HOME_NET
-
-# List of ftp servers on your network
-ipvar FTP_SERVERS $HOME_NET
-
-# List of sip servers on your network
-ipvar SIP_SERVERS $HOME_NET
-
-# List of ports you run web servers on
-portvar HTTP_PORTS [36,80,81,82,83,84,85,86,87,88,89,90,311,383,555,591,593,631,801,808,818,901,972,1158,1220,1414,1533,1741,1830,1942,2231,2301,2381,2578,2809,2980,3029,3037,3057,3128,3443,3702,4000,4343,4848,5000,5117,5250,5600,5814,6080,6173,6988,7000,7001,7005,7071,7144,7145,7510,7770,7777,7778,7779,8000,8001,8008,8014,8015,8020,8028,8040,8080,8081,8082,8085,8088,8090,8118,8123,8180,8181,8182,8222,8243,8280,8300,8333,8344,8400,8443,8500,8509,8787,8800,8888,8899,8983,9000,9002,9060,9080,9090,9091,9111,9290,9443,9447,9710,9788,9999,10000,11371,12601,13014,15489,19980,29991,33300,34412,34443,34444,40007,41080,44449,50000,50002,51423,53331,55252,55555,56712]
-
-# List of ports you want to look for SHELLCODE on.
-portvar SHELLCODE_PORTS !80
-
-# List of ports you might see oracle attacks on
-portvar ORACLE_PORTS 1024:
-
-# List of ports you want to look for SSH connections on:
-portvar SSH_PORTS 22
-
-# List of ports you run ftp servers on
-portvar FTP_PORTS [21,2100,3535]
-
-# List of ports you run SIP servers on
-portvar SIP_PORTS [5060,5061,5600]
-
-# List of file data ports for file inspection
-portvar FILE_DATA_PORTS [$HTTP_PORTS,110,143]
-
-# List of GTP ports for GTP preprocessor
-portvar GTP_PORTS [2123,2152,3386]
-
-# other variables, these should not be modified
-ipvar AIM_SERVERS [64.12.24.0/23,64.12.28.0/23,64.12.161.0/24,64.12.163.0/24,64.12.200.0/24,205.188.3.0/24,205.188.5.0/24,205.188.7.0/24,205.188.9.0/24,205.188.153.0/24,205.188.179.0/24,205.188.248.0/24]
-
-# Path to your rules files (this can be a relative path)
-# Note for Windows users: You are advised to make this an absolute path,
-# such as: c:\snort\rules
-var RULE_PATH rules
-var SO_RULE_PATH so_rules
-var PREPROC_RULE_PATH preproc_rules
-
-# If you are using reputation preprocessor set these
-var WHITE_LIST_PATH /etc/snort/rules
-var BLACK_LIST_PATH /etc/snort/rules
-
-###################################################
-# Step #2: Configure the decoder. For more information, see README.decode
-###################################################
-
-# Configure Snort to shows year in timestamps
-config show_year
-
-# Configure Snort to output timestamps in UTC
-config utc
-
-# Stop generic decode events:
-config disable_decode_alerts
-
-# Stop Alerts on experimental TCP options
-config disable_tcpopt_experimental_alerts
-
-# Stop Alerts on obsolete TCP options
-config disable_tcpopt_obsolete_alerts
-
-# Stop Alerts on T/TCP alerts
-config disable_tcpopt_ttcp_alerts
-
-# Stop Alerts on all other TCPOption type events:
-config disable_tcpopt_alerts
-
-# Stop Alerts on invalid ip options
-config disable_ipopt_alerts
-
-# Alert if value in length field (IP, TCP, UDP) is greater th elength of the packet
-# config enable_decode_oversized_alerts
-
-# Same as above, but drop packet if in Inline mode (requires enable_decode_oversized_alerts)
-# config enable_decode_oversized_drops
-
-# Configure IP / TCP checksum mode
-config checksum_mode: all
-
-# Configure maximum number of flowbit references. For more information, see README.flowbits
-# config flowbits_size: 64
-
-# Configure ports to ignore
-# config ignore_ports: tcp 21 6667:6671 1356
-# config ignore_ports: udp 1:17 53
-
-# Configure active response for non inline operation. For more information, see README.active
-# config response: eth0 attempts 2
-
-# Configure DAQ related options for inline operation. For more information, see README.daq
-#
-# config daq: <type>
-# config daq_dir: <dir>
-# config daq_mode: <mode>
-# config daq_var: <var>
-#
-# <type> ::= pcap | afpacket | dump | nfq | ipq | ipfw
-# <mode> ::= read-file | passive | inline
-# <var> ::= arbitrary <name>=<value passed to DAQ
-# <dir> ::= path as to where to look for DAQ module so's
-
-# Configure specific UID and GID to run snort as after dropping privs. For more information see snort -h command line options
-#
-# config set_gid:
-# config set_uid:
-
-# Configure default snaplen. Snort defaults to MTU of in use interface. For more information see README
-#
-# config snaplen:
-#
-
-# Configure default bpf_file to use for filtering what traffic reaches snort. For more information see snort -h command line options (-F)
-#
-# config bpf_file:
-#
-
-# Configure default log directory for snort to log to. For more information see snort -h command line options (-l)
-#
-# config logdir:
-
-
-###################################################
-# Step #3: Configure the base detection engine. For more information, see README.decode
-###################################################
-
-# Configure PCRE match limitations
-config pcre_match_limit: 3500
-config pcre_match_limit_recursion: 1500
-
-# Configure the detection engine See the Snort Manual, Configuring Snort - Includes - Config
-config detection: search-method ac-split search-optimize max-pattern-len 20
-
-# Configure the event queue. For more information, see README.event_queue
-config event_queue: max_queue 8 log 5 order_events content_length
-
-###################################################
-## Configure GTP if it is to be used.
-## For more information, see README.GTP
-####################################################
-
-# config enable_gtp
-
-###################################################
-# Per packet and rule latency enforcement
-# For more information see README.ppm
-###################################################
-
-# Per Packet latency configuration
-#config ppm: max-pkt-time 250, \
-# fastpath-expensive-packets, \
-# pkt-log
-
-# Per Rule latency configuration
-#config ppm: max-rule-time 200, \
-# threshold 3, \
-# suspend-expensive-rules, \
-# suspend-timeout 20, \
-# rule-log alert
-
-###################################################
-# Configure Perf Profiling for debugging
-# For more information see README.PerfProfiling
-###################################################
-
-#config profile_rules: print all, sort avg_ticks
-#config profile_preprocs: print all, sort avg_ticks
-
-###################################################
-# Configure protocol aware flushing
-# For more information see README.stream5
-###################################################
-config paf_max: 16000
-
-###################################################
-# Step #4: Configure dynamic loaded libraries.
-# For more information, see Snort Manual, Configuring Snort - Dynamic Modules
-###################################################
-
-# path to dynamic preprocessor libraries
-dynamicpreprocessor directory /usr/lib64/snort-2.9.8.0_dynamicpreprocessor
-
-# path to base preprocessor engine
-dynamicengine /usr/lib64/snort-2.9.8.0_dynamicengine/libsf_engine.so
-
-# path to dynamic rules libraries
-#dynamicdetection directory /usr/local/lib/snort_dynamicrules
-
-###################################################
-# Step #5: Configure preprocessors
-# For more information, see the Snort Manual, Configuring Snort - Preprocessors
-###################################################
-
-# GTP Control Channle Preprocessor. For more information, see README.GTP
-# preprocessor gtp: ports { 2123 3386 2152 }
-
-# Inline packet normalization. For more information, see README.normalize
-# Does nothing in IDS mode
-preprocessor normalize_ip4
-preprocessor normalize_tcp: ips ecn stream
-preprocessor normalize_icmp4
-preprocessor normalize_ip6
-preprocessor normalize_icmp6
-
-# Target-based IP defragmentation. For more inforation, see README.frag3
-preprocessor frag3_global: max_frags 65536
-preprocessor frag3_engine: policy windows detect_anomalies overlap_limit 10 min_fragment_length 100 timeout 180
-
-# Target-Based stateful inspection/stream reassembly. For more inforation, see README.stream5
-preprocessor stream5_global: track_tcp yes, \
- track_udp yes, \
- track_icmp no, \
- max_tcp 262144, \
- max_udp 131072, \
- max_active_responses 2, \
- min_response_seconds 5
-preprocessor stream5_tcp: policy windows, detect_anomalies, require_3whs 180, \
- overlap_limit 10, small_segments 3 bytes 150, timeout 180, \
- ports client 21 22 23 25 42 53 70 79 109 110 111 113 119 135 136 137 139 143 \
- 161 445 513 514 587 593 691 1433 1521 1741 2100 3306 6070 6665 6666 6667 6668 6669 \
- 7000 8181 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779, \
- ports both 36 80 81 82 83 84 85 86 87 88 89 90 110 311 383 443 465 563 555 591 593 631 636 801 808 818 901 972 989 992 993 994 995 1158 1220 1414 1533 1741 1830 1942 2231 2301 2381 2578 2809 2980 3029 3037 3057 3128 3443 3702 4000 4343 4848 5000 5117 5250 5600 5814 6080 6173 6988 7907 7000 7001 7005 7071 7144 7145 7510 7802 7770 7777 7778 7779 \
- 7801 7900 7901 7902 7903 7904 7905 7906 7908 7909 7910 7911 7912 7913 7914 7915 7916 \
- 7917 7918 7919 7920 8000 8001 8008 8014 8015 8020 8028 8040 8080 8081 8082 8085 8088 8090 8118 8123 8180 8181 8182 8222 8243 8280 8300 8333 8344 8400 8443 8500 8509 8787 8800 8888 8899 8983 9000 9002 9060 9080 9090 9091 9111 9290 9443 9447 9710 9788 9999 10000 11371 12601 13014 15489 19980 29991 33300 34412 34443 34444 40007 41080 44449 50000 50002 51423 53331 55252 55555 56712
-preprocessor stream5_udp: timeout 180
-
-# performance statistics. For more information, see the Snort Manual, Configuring Snort - Preprocessors - Performance Monitor
-# preprocessor perfmonitor: time 300 file /var/snort/snort.stats pktcnt 10000
-
-# HTTP normalization and anomaly detection. For more information, see README.http_inspect
-preprocessor http_inspect: global iis_unicode_map unicode.map 1252 compress_depth 65535 decompress_depth 65535
-preprocessor http_inspect_server: server default \
- http_methods { GET POST PUT SEARCH MKCOL COPY MOVE LOCK UNLOCK NOTIFY POLL BCOPY BDELETE BMOVE LINK UNLINK OPTIONS HEAD DELETE TRACE TRACK CONNECT SOURCE SUBSCRIBE UNSUBSCRIBE PROPFIND PROPPATCH BPROPFIND BPROPPATCH RPC_CONNECT PROXY_SUCCESS BITS_POST CCM_POST SMS_POST RPC_IN_DATA RPC_OUT_DATA RPC_ECHO_DATA } \
- chunk_length 500000 \
- server_flow_depth 0 \
- client_flow_depth 0 \
- post_depth 65495 \
- oversize_dir_length 500 \
- max_header_length 750 \
- max_headers 100 \
- max_spaces 200 \
- small_chunk_length { 10 5 } \
- ports { 36 80 81 82 83 84 85 86 87 88 89 90 311 383 555 591 593 631 801 808 818 901 972 1158 1220 1414 1533 1741 1830 1942 2231 2301 2381 2578 2809 2980 3029 3037 3057 3128 3443 3702 4000 4343 4848 5000 5117 5250 5600 5814 6080 6173 6988 7000 7001 7005 7071 7144 7145 7510 7770 7777 7778 7779 8000 8001 8008 8014 8015 8020 8028 8040 8080 8081 8082 8085 8088 8090 8118 8123 8180 8181 8182 8222 8243 8280 8300 8333 8344 8400 8443 8500 8509 8787 8800 8888 8899 8983 9000 9002 9060 9080 9090 9091 9111 9290 9443 9447 9710 9788 9999 10000 11371 12601 13014 15489 19980 29991 33300 34412 34443 34444 40007 41080 44449 50000 50002 51423 53331 55252 55555 56712 } \
- non_rfc_char { 0x00 0x01 0x02 0x03 0x04 0x05 0x06 0x07 } \
- enable_cookie \
- extended_response_inspection \
- inspect_gzip \
- normalize_utf \
- unlimited_decompress \
- normalize_javascript \
- apache_whitespace no \
- ascii no \
- bare_byte no \
- directory no \
- double_decode no \
- iis_backslash no \
- iis_delimiter no \
- iis_unicode no \
- multi_slash no \
- utf_8 no \
- u_encode yes \
- webroot no
-
-# ONC-RPC normalization and anomaly detection. For more information, see the Snort Manual, Configuring Snort - Preprocessors - RPC Decode
-preprocessor rpc_decode: 111 32770 32771 32772 32773 32774 32775 32776 32777 32778 32779 no_alert_multiple_requests no_alert_large_fragments no_alert_incomplete
-
-# Back Orifice detection.
-preprocessor bo
-
-# FTP / Telnet normalization and anomaly detection. For more information, see README.ftptelnet
-preprocessor ftp_telnet: global inspection_type stateful encrypted_traffic no check_encrypted
-preprocessor ftp_telnet_protocol: telnet \
- ayt_attack_thresh 20 \
- normalize ports { 23 } \
- detect_anomalies
-preprocessor ftp_telnet_protocol: ftp server default \
- def_max_param_len 100 \
- ports { 21 2100 3535 } \
- telnet_cmds yes \
- ignore_telnet_erase_cmds yes \
- ftp_cmds { ABOR ACCT ADAT ALLO APPE AUTH CCC CDUP } \
- ftp_cmds { CEL CLNT CMD CONF CWD DELE ENC EPRT } \
- ftp_cmds { EPSV ESTA ESTP FEAT HELP LANG LIST LPRT } \
- ftp_cmds { LPSV MACB MAIL MDTM MIC MKD MLSD MLST } \
- ftp_cmds { MODE NLST NOOP OPTS PASS PASV PBSZ PORT } \
- ftp_cmds { PROT PWD QUIT REIN REST RETR RMD RNFR } \
- ftp_cmds { RNTO SDUP SITE SIZE SMNT STAT STOR STOU } \
- ftp_cmds { STRU SYST TEST TYPE USER XCUP XCRC XCWD } \
- ftp_cmds { XMAS XMD5 XMKD XPWD XRCP XRMD XRSQ XSEM } \
- ftp_cmds { XSEN XSHA1 XSHA256 } \
- alt_max_param_len 0 { ABOR CCC CDUP ESTA FEAT LPSV NOOP PASV PWD QUIT REIN STOU SYST XCUP XPWD } \
- alt_max_param_len 200 { ALLO APPE CMD HELP NLST RETR RNFR STOR STOU XMKD } \
- alt_max_param_len 256 { CWD RNTO } \
- alt_max_param_len 400 { PORT } \
- alt_max_param_len 512 { SIZE } \
- chk_str_fmt { ACCT ADAT ALLO APPE AUTH CEL CLNT CMD } \
- chk_str_fmt { CONF CWD DELE ENC EPRT EPSV ESTP HELP } \
- chk_str_fmt { LANG LIST LPRT MACB MAIL MDTM MIC MKD } \
- chk_str_fmt { MLSD MLST MODE NLST OPTS PASS PBSZ PORT } \
- chk_str_fmt { PROT REST RETR RMD RNFR RNTO SDUP SITE } \
- chk_str_fmt { SIZE SMNT STAT STOR STRU TEST TYPE USER } \
- chk_str_fmt { XCRC XCWD XMAS XMD5 XMKD XRCP XRMD XRSQ } \
- chk_str_fmt { XSEM XSEN XSHA1 XSHA256 } \
- cmd_validity ALLO < int [ char R int ] > \
- cmd_validity EPSV < [ { char 12 | char A char L char L } ] > \
- cmd_validity MACB < string > \
- cmd_validity MDTM < [ date nnnnnnnnnnnnnn[.n[n[n]]] ] string > \
- cmd_validity MODE < char ASBCZ > \
- cmd_validity PORT < host_port > \
- cmd_validity PROT < char CSEP > \
- cmd_validity STRU < char FRPO [ string ] > \
- cmd_validity TYPE < { char AE [ char NTC ] | char I | char L [ number ] } >
-preprocessor ftp_telnet_protocol: ftp client default \
- max_resp_len 256 \
- bounce yes \
- ignore_telnet_erase_cmds yes \
- telnet_cmds yes
-
-
-# SMTP normalization and anomaly detection. For more information, see README.SMTP
-preprocessor smtp: ports { 25 465 587 691 } \
- inspection_type stateful \
- b64_decode_depth 0 \
- qp_decode_depth 0 \
- bitenc_decode_depth 0 \
- uu_decode_depth 0 \
- log_mailfrom \
- log_rcptto \
- log_filename \
- log_email_hdrs \
- normalize cmds \
- normalize_cmds { ATRN AUTH BDAT CHUNKING DATA DEBUG EHLO EMAL ESAM ESND ESOM ETRN EVFY } \
- normalize_cmds { EXPN HELO HELP IDENT MAIL NOOP ONEX QUEU QUIT RCPT RSET SAML SEND SOML } \
- normalize_cmds { STARTTLS TICK TIME TURN TURNME VERB VRFY X-ADAT X-DRCP X-ERCP X-EXCH50 } \
- normalize_cmds { X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
- max_command_line_len 512 \
- max_header_line_len 1000 \
- max_response_line_len 512 \
- alt_max_command_line_len 260 { MAIL } \
- alt_max_command_line_len 300 { RCPT } \
- alt_max_command_line_len 500 { HELP HELO ETRN EHLO } \
- alt_max_command_line_len 255 { EXPN VRFY ATRN SIZE BDAT DEBUG EMAL ESAM ESND ESOM EVFY IDENT NOOP RSET } \
- alt_max_command_line_len 246 { SEND SAML SOML AUTH TURN ETRN DATA RSET QUIT ONEX QUEU STARTTLS TICK TIME TURNME VERB X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
- valid_cmds { ATRN AUTH BDAT CHUNKING DATA DEBUG EHLO EMAL ESAM ESND ESOM ETRN EVFY } \
- valid_cmds { EXPN HELO HELP IDENT MAIL NOOP ONEX QUEU QUIT RCPT RSET SAML SEND SOML } \
- valid_cmds { STARTTLS TICK TIME TURN TURNME VERB VRFY X-ADAT X-DRCP X-ERCP X-EXCH50 } \
- valid_cmds { X-EXPS X-LINK2STATE XADR XAUTH XCIR XEXCH50 XGEN XLICENSE XQUE XSTA XTRN XUSR } \
- xlink2state { enabled }
-
-# Portscan detection. For more information, see README.sfportscan
-preprocessor sfportscan: proto { all } memcap { 10000000 } sense_level { low }
-
-# ARP spoof detection. For more information, see the Snort Manual - Configuring Snort - Preprocessors - ARP Spoof Preprocessor
-# preprocessor arpspoof
-# preprocessor arpspoof_detect_host: 192.168.40.1 f0:0f:00:f0:0f:00
-
-# SSH anomaly detection. For more information, see README.ssh
-preprocessor ssh: server_ports { 22 } \
- autodetect \
- max_client_bytes 19600 \
- max_encrypted_packets 20 \
- max_server_version_len 100 \
- enable_respoverflow enable_ssh1crc32 \
- enable_srvoverflow enable_protomismatch
-
-# SMB / DCE-RPC normalization and anomaly detection. For more information, see README.dcerpc2
-preprocessor dcerpc2: memcap 102400, events [co ]
-preprocessor dcerpc2_server: default, policy WinXP, \
- detect [smb [139,445], tcp 135, udp 135, rpc-over-http-server 593], \
- autodetect [tcp 1025:, udp 1025:, rpc-over-http-server 1025:], \
- smb_max_chain 3, smb_invalid_shares ["C$", "D$", "ADMIN$"]
-
-# DNS anomaly detection. For more information, see README.dns
-preprocessor dns: ports { 53 } enable_rdata_overflow
-
-# SSL anomaly detection and traffic bypass. For more information, see README.ssl
-preprocessor ssl: ports { 443 465 563 636 989 992 993 994 995 5061 7801 7802 7900 7901 7902 7903 7904 7905 7906 7907 7908 7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 }, trustservers, noinspect_encrypted
-
-# SDF sensitive data preprocessor. For more information see README.sensitive_data
-preprocessor sensitive_data: alert_threshold 25
-
-# SIP Session Initiation Protocol preprocessor. For more information see README.sip
-preprocessor sip: max_sessions 40000, \
- ports { 5060 5061 5600 }, \
- methods { invite \
- cancel \
- ack \
- bye \
- register \
- options \
- refer \
- subscribe \
- update \
- join \
- info \
- message \
- notify \
- benotify \
- do \
- qauth \
- sprack \
- publish \
- service \
- unsubscribe \
- prack }, \
- max_uri_len 512, \
- max_call_id_len 80, \
- max_requestName_len 20, \
- max_from_len 256, \
- max_to_len 256, \
- max_via_len 1024, \
- max_contact_len 512, \
- max_content_len 2048
-
-# IMAP preprocessor. For more information see README.imap
-preprocessor imap: \
- ports { 143 } \
- b64_decode_depth 0 \
- qp_decode_depth 0 \
- bitenc_decode_depth 0 \
- uu_decode_depth 0
-
-# POP preprocessor. For more information see README.pop
-preprocessor pop: \
- ports { 110 } \
- b64_decode_depth 0 \
- qp_decode_depth 0 \
- bitenc_decode_depth 0 \
- uu_decode_depth 0
-
-# Modbus preprocessor. For more information see README.modbus
-preprocessor modbus: ports { 502 }
-
-# DNP3 preprocessor. For more information see README.dnp3
-preprocessor dnp3: ports { 20000 } \
- memcap 262144 \
- check_crc
-
-# Reputation preprocessor. For more information see README.reputation
-preprocessor reputation: \
- memcap 500, \
- priority whitelist, \
- nested_ip inner, \
- whitelist $WHITE_LIST_PATH/white_list.rules, \
- blacklist $BLACK_LIST_PATH/black_list.rules
-
-###################################################
-# Step #6: Configure output plugins
-# For more information, see Snort Manual, Configuring Snort - Output Modules
-###################################################
-
-# unified2
-# Recommended for most installs
-# output unified2: filename merged.log, limit 128, nostamp, mpls_event_types, vlan_event_types
-
-# Additional configuration for specific types of installs
-# output alert_unified2: filename snort.alert, limit 128, nostamp
-# output log_unified2: filename snort.log, limit 128, nostamp
-
-# syslog
-# output alert_syslog: LOG_AUTH LOG_ALERT
-
-# pcap
-# output log_tcpdump: tcpdump.log
-
-# metadata reference data. do not modify these lines
-include classification.config
-include reference.config
-
-
-###################################################
-# Step #7: Customize your rule set
-# For more information, see Snort Manual, Writing Snort Rules
-#
-# NOTE: All categories are enabled in this conf file
-###################################################
-
-include $RULE_PATH/community.rules
-
-# site specific rules
-# include $RULE_PATH/local.rules
-# include $RULE_PATH/app-detect.rules
-# include $RULE_PATH/attack-responses.rules
-# include $RULE_PATH/backdoor.rules
-# include $RULE_PATH/bad-traffic.rules
-# include $RULE_PATH/blacklist.rules
-# include $RULE_PATH/botnet-cnc.rules
-# include $RULE_PATH/browser-chrome.rules
-# include $RULE_PATH/browser-firefox.rules
-# include $RULE_PATH/browser-ie.rules
-# include $RULE_PATH/browser-other.rules
-# include $RULE_PATH/browser-plugins.rules
-# include $RULE_PATH/browser-webkit.rules
-# include $RULE_PATH/chat.rules
-# include $RULE_PATH/content-replace.rules
-# include $RULE_PATH/ddos.rules
-# include $RULE_PATH/dns.rules
-# include $RULE_PATH/dos.rules
-# include $RULE_PATH/experimental.rules
-# include $RULE_PATH/exploit-kit.rules
-# include $RULE_PATH/exploit.rules
-# include $RULE_PATH/file-executable.rules
-# include $RULE_PATH/file-flash.rules
-# include $RULE_PATH/file-identify.rules
-# include $RULE_PATH/file-image.rules
-# include $RULE_PATH/file-java.rules
-# include $RULE_PATH/file-multimedia.rules
-# include $RULE_PATH/file-office.rules
-# include $RULE_PATH/file-other.rules
-# include $RULE_PATH/file-pdf.rules
-# include $RULE_PATH/finger.rules
-# include $RULE_PATH/ftp.rules
-# include $RULE_PATH/icmp-info.rules
-# include $RULE_PATH/icmp.rules
-# include $RULE_PATH/imap.rules
-# include $RULE_PATH/indicator-compromise.rules
-# include $RULE_PATH/indicator-obfuscation.rules
-# include $RULE_PATH/indicator-scan.rules
-# include $RULE_PATH/indicator-shellcode.rules
-# include $RULE_PATH/info.rules
-# include $RULE_PATH/malware-backdoor.rules
-# include $RULE_PATH/malware-cnc.rules
-# include $RULE_PATH/malware-other.rules
-# include $RULE_PATH/malware-tools.rules
-# include $RULE_PATH/misc.rules
-# include $RULE_PATH/multimedia.rules
-# include $RULE_PATH/mysql.rules
-# include $RULE_PATH/netbios.rules
-# include $RULE_PATH/nntp.rules
-# include $RULE_PATH/oracle.rules
-# include $RULE_PATH/os-linux.rules
-# include $RULE_PATH/os-mobile.rules
-# include $RULE_PATH/os-other.rules
-# include $RULE_PATH/os-solaris.rules
-# include $RULE_PATH/os-windows.rules
-# include $RULE_PATH/other-ids.rules
-# include $RULE_PATH/p2p.rules
-# include $RULE_PATH/phishing-spam.rules
-# include $RULE_PATH/policy-multimedia.rules
-# include $RULE_PATH/policy-other.rules
-# include $RULE_PATH/policy.rules
-# include $RULE_PATH/policy-social.rules
-# include $RULE_PATH/policy-spam.rules
-# include $RULE_PATH/pop2.rules
-# include $RULE_PATH/pop3.rules
-# include $RULE_PATH/protocol-dns.rules
-# include $RULE_PATH/protocol-finger.rules
-# include $RULE_PATH/protocol-ftp.rules
-# include $RULE_PATH/protocol-icmp.rules
-# include $RULE_PATH/protocol-imap.rules
-# include $RULE_PATH/protocol-nntp.rules
-# include $RULE_PATH/protocol-other.rules
-# include $RULE_PATH/protocol-pop.rules
-# include $RULE_PATH/protocol-rpc.rules
-# include $RULE_PATH/protocol-scada.rules
-# include $RULE_PATH/protocol-services.rules
-# include $RULE_PATH/protocol-snmp.rules
-# include $RULE_PATH/protocol-telnet.rules
-# include $RULE_PATH/protocol-tftp.rules
-# include $RULE_PATH/protocol-voip.rules
-# include $RULE_PATH/pua-adware.rules
-# include $RULE_PATH/pua-other.rules
-# include $RULE_PATH/pua-p2p.rules
-# include $RULE_PATH/pua-toolbars.rules
-# include $RULE_PATH/rpc.rules
-# include $RULE_PATH/rservices.rules
-# include $RULE_PATH/scada.rules
-# include $RULE_PATH/scan.rules
-# include $RULE_PATH/server-apache.rules
-# include $RULE_PATH/server-iis.rules
-# include $RULE_PATH/server-mail.rules
-# include $RULE_PATH/server-mssql.rules
-# include $RULE_PATH/server-oracle.rules
-# include $RULE_PATH/server-other.rules
-# include $RULE_PATH/server-samba.rules
-# include $RULE_PATH/server-webapp.rules
-# include $RULE_PATH/shellcode.rules
-# include $RULE_PATH/smtp.rules
-# include $RULE_PATH/snmp.rules
-# include $RULE_PATH/specific-threats.rules
-# include $RULE_PATH/spyware-put.rules
-# include $RULE_PATH/sql.rules
-# include $RULE_PATH/telnet.rules
-# include $RULE_PATH/tftp.rules
-# include $RULE_PATH/virus.rules
-# include $RULE_PATH/voip.rules
-# include $RULE_PATH/web-activex.rules
-# include $RULE_PATH/web-attacks.rules
-# include $RULE_PATH/web-cgi.rules
-# include $RULE_PATH/web-client.rules
-# include $RULE_PATH/web-coldfusion.rules
-# include $RULE_PATH/web-frontpage.rules
-# include $RULE_PATH/web-iis.rules
-# include $RULE_PATH/web-misc.rules
-# include $RULE_PATH/web-php.rules
-# include $RULE_PATH/x11.rules
-
-###################################################
-# Step #8: Customize your preprocessor and decoder alerts
-# For more information, see README.decoder_preproc_rules
-###################################################
-
-# decoder and preprocessor event rules
-# include $PREPROC_RULE_PATH/preprocessor.rules
-# include $PREPROC_RULE_PATH/decoder.rules
-# include $PREPROC_RULE_PATH/sensitive-data.rules
-
-###################################################
-# Step #9: Customize your Shared Object Snort Rules
-# For more information, see http://vrt-blog.snort.org/2009/01/using-vrt-certified-shared-object-rules.html
-###################################################
-
-# dynamic library rules
-# include $SO_RULE_PATH/browser-ie.rules
-# include $SO_RULE_PATH/browser-other.rules
-# include $SO_RULE_PATH/exploit-kit.rules
-# include $SO_RULE_PATH/file-flash.rules
-# include $SO_RULE_PATH/file-image.rules
-# include $SO_RULE_PATH/file-java.rules
-# include $SO_RULE_PATH/file-multimedia.rules
-# include $SO_RULE_PATH/file-office.rules
-# include $SO_RULE_PATH/file-other.rules
-# include $SO_RULE_PATH/file-pdf.rules
-# include $SO_RULE_PATH/indicator-shellcode.rules
-# include $SO_RULE_PATH/malware-cnc.rules
-# include $SO_RULE_PATH/malware-other.rules
-# include $SO_RULE_PATH/netbios.rules
-# include $SO_RULE_PATH/os-linux.rules
-# include $SO_RULE_PATH/os-other.rules
-# include $SO_RULE_PATH/os-windows.rules
-# include $SO_RULE_PATH/policy-social.rules
-# include $SO_RULE_PATH/protocol-dns.rules
-# include $SO_RULE_PATH/protocol-nntp.rules
-# include $SO_RULE_PATH/protocol-other.rules
-# include $SO_RULE_PATH/protocol-snmp.rules
-# include $SO_RULE_PATH/protocol-voip.rules
-# include $SO_RULE_PATH/pua-p2p.rules
-# include $SO_RULE_PATH/server-apache.rules
-# include $SO_RULE_PATH/server-iis.rules
-# include $SO_RULE_PATH/server-mail.rules
-# include $SO_RULE_PATH/server-oracle.rules
-# include $SO_RULE_PATH/server-other.rules
-# include $SO_RULE_PATH/server-webapp.rules
-
-# legacy dynamic library rule files
-# include $SO_RULE_PATH/bad-traffic.rules
-# include $SO_RULE_PATH/browser-ie.rules
-# include $SO_RULE_PATH/chat.rules
-# include $SO_RULE_PATH/dos.rules
-# include $SO_RULE_PATH/exploit.rules
-# include $SO_RULE_PATH/file-flash.rules
-# include $SO_RULE_PATH/icmp.rules
-# include $SO_RULE_PATH/imap.rules
-# include $SO_RULE_PATH/misc.rules
-# include $SO_RULE_PATH/multimedia.rules
-# include $SO_RULE_PATH/netbios.rules
-# include $SO_RULE_PATH/nntp.rules
-# include $SO_RULE_PATH/p2p.rules
-# include $SO_RULE_PATH/smtp.rules
-# include $SO_RULE_PATH/snmp.rules
-# include $SO_RULE_PATH/specific-threats.rules
-# include $SO_RULE_PATH/web-activex.rules
-# include $SO_RULE_PATH/web-client.rules
-# include $SO_RULE_PATH/web-iis.rules
-# include $SO_RULE_PATH/web-misc.rules
-
-# Event thresholding or suppression commands. See threshold.conf
-include threshold.conf
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/meta/main.yml b/metron-deployment/roles/snort/meta/main.yml
deleted file mode 100644
index ebca8ed..0000000
--- a/metron-deployment/roles/snort/meta/main.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - epel
- - libselinux-python
- - build-tools
- - kafka-client
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/tasks/daq.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/daq.yml b/metron-deployment/roles/snort/tasks/daq.yml
deleted file mode 100644
index c8bd4b0..0000000
--- a/metron-deployment/roles/snort/tasks/daq.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download daq
- get_url:
- url: "{{ dag_src_url }}"
- dest: "/tmp/daq-{{ daq_version }}.src.rpm"
-
-- name: Build daq
- shell: "rpmbuild --rebuild daq-{{ daq_version }}.src.rpm"
- args:
- chdir: /tmp
- creates: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
-
-- name: Install daq
- yum:
- name: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/main.yml b/metron-deployment/roles/snort/tasks/main.yml
deleted file mode 100644
index 4736bee..0000000
--- a/metron-deployment/roles/snort/tasks/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: daq.yml
-- include: snort.yml
-- include: producer.yml
-- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/nic.yml b/metron-deployment/roles/snort/tasks/nic.yml
deleted file mode 100644
index 6053618..0000000
--- a/metron-deployment/roles/snort/tasks/nic.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/tasks/producer.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/producer.yml b/metron-deployment/roles/snort/tasks/producer.yml
deleted file mode 100644
index d6e9c3a..0000000
--- a/metron-deployment/roles/snort/tasks/producer.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Snort producer home directory
- file:
- path: "{{ snort_producer_home }}"
- state: directory
- mode: 0755
-
-- name: Install Snort producer start script
- template: src=start-snort-producer.sh dest={{ snort_producer_home }}/start-snort-producer.sh mode=0755
-
-- name: Install init.d service script
- template: src=snort-producer dest=/etc/init.d/snort-producer mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/tasks/snort.yml b/metron-deployment/roles/snort/tasks/snort.yml
deleted file mode 100644
index de26936..0000000
--- a/metron-deployment/roles/snort/tasks/snort.yml
+++ /dev/null
@@ -1,85 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download snort
- get_url:
- url: "{{ snort_src_url }}"
- dest: "/tmp/snort-{{ snort_version }}.src.rpm"
-
-- name: Build snort
- shell: "rpmbuild --rebuild snort-{{ snort_version }}.src.rpm"
- args:
- chdir: /tmp
- creates: /root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
-
-- name: Install snort
- yum:
- name: /root/rpmbuild/RPMS/x86_64/snort-{{ snort_version }}.x86_64.rpm
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Download snort community rules
- get_url:
- url: "{{ snort_community_rules_url }}"
- dest: "/tmp/community-rules.tar.gz"
-
-- name: Extract tarball
- unarchive:
- src: "/tmp/community-rules.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/community-rules"
-
-- name: Install snort rules
- shell: "{{ item }}"
- args:
- chdir: /tmp
- with_items:
- - cp -r community-rules/community.rules /etc/snort/rules
- - touch /etc/snort/rules/white_list.rules
- - touch /etc/snort/rules/black_list.rules
- - touch /var/log/snort/alerts
- - chown -R snort:snort /etc/snort
-
-- name: Uncomment all snort community rules
- shell: sed -i 's/^# alert/alert/' /etc/snort/rules/community.rules
-
-- name: Download snort configuration
- copy: src=snort.conf dest=/etc/snort/snort.conf
-
-- name: Configure home network
- lineinfile:
- dest: /etc/snort/snort.conf
- regexp: "^ipvar HOME_NET.*$"
- line: "ipvar HOME_NET {{ snort_home_net }}"
-
-- name: Configure alerting
- lineinfile:
- dest: /etc/snort/snort.conf
- line: "output alert_csv: {{ snort_alert_csv_path }} default"
-
-- name: Configure sysconfig
- lineinfile:
- dest: /etc/sysconfig/snort
- regexp: "{{ item.regexp }}"
- line: "{{ item.line }}"
- with_items:
- - { regexp: "^ALERTMODE=.*$", line: "ALERTMODE=" }
- - { regexp: "^NO_PACKET_LOG=.*$", line: "NO_PACKET_LOG=1" }
- - { regexp: "^INTERFACE=.*$", line: "INTERFACE={{ sniff_interface }}" }
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/templates/snort-producer
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/templates/snort-producer b/metron-deployment/roles/snort/templates/snort-producer
deleted file mode 100644
index 1cb68ca..0000000
--- a/metron-deployment/roles/snort/templates/snort-producer
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Snort Kafka producer daemon
-# chkconfig: 345 20 80
-# description: Runs Snort Kafka producer
-# processname: snort-producer
-#
-NAME=snort-producer
-DESC="Executes Snort Kafka producer"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-DAEMON_PATH="{{ snort_producer_home }}"
-DAEMON="{{ snort_producer_start }}"
-DAEMONOPTS="${@:2}"
-
-case "$1" in
- start)
- printf "%-50s" "Starting $NAME..."
-
- # kick-off the daemon
- cd $DAEMON_PATH
- PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- ;;
-
- status)
- printf "%-50s" "Checking $NAME..."
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- echo "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- stop)
- printf "%-50s" "Stopping $NAME"
- PID=`cat $PIDFILE`
- PGID=`ps -o pgid= $PID | xargs`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- kill -- -$PGID
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/snort/templates/start-snort-producer.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/snort/templates/start-snort-producer.sh b/metron-deployment/roles/snort/templates/start-snort-producer.sh
deleted file mode 100644
index 5234aae..0000000
--- a/metron-deployment/roles/snort/templates/start-snort-producer.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# a simple script that tails the Snort alert log and pipes alerts into the
-# snort Kafka topic via the Kafka console producer
-#
-tail -F {{ snort_alert_csv_path }} | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ snort_topic }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/tap_interface/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/tap_interface/defaults/main.yml b/metron-deployment/roles/tap_interface/defaults/main.yml
deleted file mode 100644
index ca752b4..0000000
--- a/metron-deployment/roles/tap_interface/defaults/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-tap_if: tap0
-tap_ip: 10.0.0.1
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/tap_interface/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/tap_interface/tasks/main.yml b/metron-deployment/roles/tap_interface/tasks/main.yml
deleted file mode 100644
index 1de3abe..0000000
--- a/metron-deployment/roles/tap_interface/tasks/main.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install tunctl
- yum:
- name: tunctl
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Create {{ tap_if }}
- command: tunctl -p
-
-- name: Bring up {{ tap_if }} on {{ tap_ip }}
- command: ifconfig {{ tap_if }} {{ tap_ip }} up
-
-- name: Put {{ tap_if }} in PROMISC
- command: ip link set {{ tap_if }} promisc on
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/defaults/main.yml b/metron-deployment/roles/yaf/defaults/main.yml
deleted file mode 100644
index 0a0394b..0000000
--- a/metron-deployment/roles/yaf/defaults/main.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-fixbuf_version: 1.7.1
-yaf_version: 2.8.0
-yaf_home: /opt/yaf
-yaf_topic: yaf
-yaf_bin: /usr/local/bin/yaf
-yafscii_bin: /usr/local/bin/yafscii
-yaf_log: /var/log/yaf.log
-kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh
-daemon_bin: /usr/local/bin/airdaemon
-yaf_start: /opt/yaf/start-yaf.sh
-yaf_args:
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/meta/main.yml b/metron-deployment/roles/yaf/meta/main.yml
deleted file mode 100644
index 768bcac..0000000
--- a/metron-deployment/roles/yaf/meta/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - build-tools
- - java_jdk
- - libselinux-python
- - kafka-client
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/tasks/fixbuf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/fixbuf.yml b/metron-deployment/roles/yaf/tasks/fixbuf.yml
deleted file mode 100644
index 9cd9244..0000000
--- a/metron-deployment/roles/yaf/tasks/fixbuf.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download fixbuf
- get_url:
- url: "http://tools.netsa.cert.org/releases/libfixbuf-{{fixbuf_version}}.tar.gz"
- dest: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
-
-- name: Extract fixbuf tarball
- unarchive:
- src: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/libfixbuf-{{fixbuf_version}}"
-
-- name: Compile and Install fixbuf
- shell: "{{item}}"
- args:
- chdir: "/tmp/libfixbuf-{{fixbuf_version}}"
- with_items:
- - ./configure
- - make
- - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/main.yml b/metron-deployment/roles/yaf/tasks/main.yml
deleted file mode 100644
index 7d21348..0000000
--- a/metron-deployment/roles/yaf/tasks/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: fixbuf.yml
-- include: yaf.yml
-- include: nic.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/tasks/nic.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/nic.yml b/metron-deployment/roles/yaf/tasks/nic.yml
deleted file mode 100644
index 6053618..0000000
--- a/metron-deployment/roles/yaf/tasks/nic.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Turn on promiscuous mode for {{ sniff_interface }}
- shell: "ip link set {{ sniff_interface }} promisc on"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/tasks/yaf.yml b/metron-deployment/roles/yaf/tasks/yaf.yml
deleted file mode 100644
index 21445b2..0000000
--- a/metron-deployment/roles/yaf/tasks/yaf.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download yaf
- get_url:
- url: "http://tools.netsa.cert.org/releases/yaf-{{yaf_version}}.tar.gz"
- dest: "/tmp/yaf-{{yaf_version}}.tar.gz"
-
-- name: Extract yaf tarball
- unarchive:
- src: "/tmp/yaf-{{yaf_version}}.tar.gz"
- dest: /tmp
- copy: no
- creates: /usr/local/bin/yaf
-
-- name: Compile and install yaf
- shell: "{{item}}"
- args:
- chdir: "/tmp/yaf-{{yaf_version}}"
- creates: /usr/local/bin/yaf
- with_items:
- - ./configure --enable-applabel --enable-plugins
- - make
- - make install
-
-- name: Create yaf home directory
- file:
- path: "{{ yaf_home }}"
- state: directory
- mode: 0755
-
-- name: Install yaf start script
- template: src=start-yaf.sh dest={{ yaf_home }}/start-yaf.sh mode=0755
-
-- name: Install init.d service script
- template: src=yaf dest=/etc/init.d/yaf mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/templates/start-yaf.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/templates/start-yaf.sh b/metron-deployment/roles/yaf/templates/start-yaf.sh
deleted file mode 100644
index 9660e72..0000000
--- a/metron-deployment/roles/yaf/templates/start-yaf.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# a very simply metron probe that captures the output of yaf - yet another
-# flowmeter - and sends the output to kafka so that it can be consumed
-# by metron
-#
-{{ yaf_bin }} --in {{ sniff_interface }} --live pcap "${@:1}" | {{ yafscii_bin }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/yaf/templates/yaf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/yaf/templates/yaf b/metron-deployment/roles/yaf/templates/yaf
deleted file mode 100644
index 18bc4ac..0000000
--- a/metron-deployment/roles/yaf/templates/yaf
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# yaf daemon
-# chkconfig: 345 20 80
-# description: Runs yaf - yet another flowmeter
-# processname: yaf
-#
-NAME=yaf
-DESC="Executes yaf - yet another flowmeter"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-DAEMON_PATH="{{ yaf_home }}"
-DAEMON="{{ yaf_start }}"
-DAEMONOPTS="${@:2}"
-
-case "$1" in
- start)
- printf "%-50s" "Starting $NAME..."
-
- # kick-off the daemon
- cd $DAEMON_PATH
- PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- ;;
-
- status)
- printf "%-50s" "Checking $NAME..."
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- echo "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- stop)
- printf "%-50s" "Stopping $NAME"
- PID=`cat $PIDFILE`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- kill -HUP $PID
- killall $NAME
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
[15/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml b/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
deleted file mode 100644
index be9b1d3..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/copy_bundles.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Copy Metron bundles
- copy:
- src: "{{ item }}"
- dest: "{{ metron_directory }}"
- with_items:
- - "{{ metron_solr_bundle_path }}"
- - "{{ metron_elasticsearch_bundle_path }}"
- - "{{ metron_enrichment_bundle_path }}"
- - "{{ metron_indexing_bundle_path }}"
- - "{{ metron_parsers_bundle_path }}"
- - "{{ metron_data_management_bundle_path }}"
- - "{{ metron_common_bundle_path }}"
- - "{{ metron_pcap_bundle_path }}"
- - "{{ metron_maas_bundle_path }}"
- - "{{ metron_profiler_bundle_path }}"
- - "{{ metron_profiler_client_bundle_path }}"
-
-- name: Unbundle Metron bundles
- shell: "{{ item }}"
- args:
- chdir: "{{ metron_directory }}"
- with_items:
- - "tar xzvf {{ metron_solr_bundle_name }}"
- - "tar xzvf {{ metron_elasticsearch_bundle_name }}"
- - "tar xzvf {{ metron_enrichment_bundle_name }}"
- - "tar xzvf {{ metron_indexing_bundle_name }}"
- - "tar xzvf {{ metron_parsers_bundle_name }}"
- - "tar xzvf {{ metron_data_management_bundle_name }}"
- - "tar xzvf {{ metron_common_bundle_name }}"
- - "tar xzvf {{ metron_pcap_bundle_name }}"
- - "tar xzvf {{ metron_maas_bundle_name }}"
- - "tar xzvf {{ metron_profiler_bundle_name }}"
- - "tar xvzf {{ metron_profiler_bundle_name }} "
- - "tar xvzf {{ metron_profiler_client_bundle_name }} "
- - rm *.tar.gz
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/es_purge.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/es_purge.yml b/metron-deployment/roles/metron_streaming/tasks/es_purge.yml
deleted file mode 100644
index 22616ca..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/es_purge.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Empty Log Files for ES Purge
- file:
- path: "{{ item }}"
- state: touch
- owner: hdfs
- group: hdfs
- mode: 0644
- with_items:
- - /var/log/bro-purge/cron-es-bro-purge.log
- - /var/log/yaf-purge/cron-es-yaf-purge.log
- - /var/log/snort-purge/cron-es-snort-purge.log
-
-
-- name: Purge Elasticsearch Indices every 30 days.
- cron:
- name: "{{ item.name }}"
- job: "{{ item.job }}"
- special_time: daily
- user: hdfs
- with_items:
- - { name: "bro_es_purge", job: "{{ es_bro_purge_cronjob }}" }
- - { name: "yaf_es_purge", job: "{{ es_yaf_purge_cronjob }}" }
- - { name: "snort_es_purge", job: "{{ es_snort_purge_cronjob }}" }
-
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/geoip.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/geoip.yml b/metron-deployment/roles/metron_streaming/tasks/geoip.yml
deleted file mode 100644
index b26f889..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/geoip.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Load Geo DB
- shell: "{{ metron_directory }}/bin/geo_enrichment_load.sh -z {{ zookeeper_url }}"
- become: yes
- become_user: hdfs
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml b/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml
deleted file mode 100644
index d857bf5..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/grok_upload.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create HDFS directory for grok patterns
- command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}/patterns
- become: yes
- become_user: hdfs
-
-- name: Assign hfds user as owner of {{ metron_hdfs_output_dir }}/patterns HDFS directory
- command: hdfs dfs -chown -R hdfs:hadoop {{ metron_hdfs_output_dir }}/patterns
- become: yes
- become_user: hdfs
-
-- name: Assign permissions of HDFS {{ metron_hdfs_output_dir }}/patterns directory
- command: hdfs dfs -chmod -R 775 {{ metron_hdfs_output_dir }}/patterns
- become: yes
- become_user: hdfs
-
-- name: Upload Grok Patterns to hdfs://{{ metron_hdfs_output_dir }}
- command: hdfs dfs -put -f {{ metron_directory }}/patterns {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
deleted file mode 100644
index 125d41d..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/hdfs_filesystem.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create root user HDFS directory
- command: hdfs dfs -mkdir -p /user/root
- become: yes
- become_user: hdfs
-
-- name: Assign root as owner of /user/root HDFS directory
- command: hdfs dfs -chown root:root /user/root
- become: yes
- become_user: hdfs
-
-- name: Create Metron HDFS output directory
- command: hdfs dfs -mkdir -p {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
-- name: Create Metron HDFS geo directory
- command: hdfs dfs -mkdir -p {{ geo_hdfs_path }}
- become: yes
- become_user: hdfs
-
-- name: Assign hdfs as owner of HDFS output directory
- command: hdfs dfs -chown hdfs:hadoop {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
-- name: Assign hdfs as owner of HDFS Geo directory
- command: hdfs dfs -chown hdfs:hadoop {{ geo_hdfs_path }}
- become: yes
- become_user: hdfs
-
-- name: Assign permissions of HDFS output directory
- command: hdfs dfs -chmod 775 {{ metron_hdfs_output_dir }}
- become: yes
- become_user: hdfs
-
-- name: Assign permissions of HDFS geo directory
- command: hdfs dfs -chmod 775 {{ geo_hdfs_path }}
- become: yes
- become_user: hdfs
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml b/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml
deleted file mode 100644
index 33442e4..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/hdfs_purge.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Log Directories for HDFS Purge
- file:
- path: "{{ item }}"
- state: directory
- mode: 0755
- owner: hdfs
- group: hdfs
- with_items:
- - /var/log/bro-purge
- - /var/log/yaf-purge
- - /var/log/snort-purge
-
-- name: Create Empty Log Files for HDFS Purge
- file:
- path: "{{ item }}"
- state: touch
- owner: hdfs
- group: hdfs
- mode: 0644
- with_items:
- - /var/log/bro-purge/cron-hdfs-bro-purge.log
- - /var/log/yaf-purge/cron-hdfs-yaf-purge.log
- - /var/log/snort-purge/cron-hdfs-snort-purge.log
-
-- name: Purge HDFS Sensor Data every 30 days.
- cron:
- name: "{{ item.name }}"
- job: "{{ item.job }}"
- special_time: daily
- user: hdfs
- with_items:
- - { name: "bro_hdfs_purge", job: "{{ hdfs_bro_purge_cronjob }}" }
- - { name: "yaf_hdfs_purge", job: "{{ hdfs_yaf_purge_cronjob }}" }
- - { name: "snort_hdfs_purge", job: "{{ hdfs_snort_purge_cronjob }}" }
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/main.yml b/metron-deployment/roles/metron_streaming/tasks/main.yml
deleted file mode 100644
index 498b8dd..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/main.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Metron directories
- file: path={{ metron_directory }}/{{ item }} state=directory mode=0755
- with_items:
- - lib
- - bin
- - config
-
-- name: Create Source Config Directory
- file:
- path: "{{ zookeeper_config_path }}"
- state: directory
- mode: 0755
-
-- include: copy_bundles.yml
-
-- include: hdfs_filesystem.yml
- run_once: true
-
-- include: grok_upload.yml
- run_once: true
-
-- include: topologies.yml
-
-- include: source_config.yml
- run_once: true
-
-- include: geoip.yml
- run_once: true
-
-- include: threat_intel.yml
- run_once: true
- when: threat_intel_bulk_load == True
-
-- include: hdfs_purge.yml
-
-- include: es_purge.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/source_config.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/source_config.yml b/metron-deployment/roles/metron_streaming/tasks/source_config.yml
deleted file mode 100644
index 1c967bd..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/source_config.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Copy Elasticsearch Global Config File
- template:
- src: "templates/config/elasticsearch.global.json"
- dest: "{{ zookeeper_global_config_path }}"
- mode: 0644
- when: install_elasticsearch | default(False) == True
-
-- name: Copy Solr Global Config File
- template:
- src: "../roles/metron_streaming/templates/config/solr.global.json"
- dest: "{{ zookeeper_global_config_path }}"
- mode: 0644
- when: install_solr | default(False) == True
-
-- name: Load Config
- shell: "{{ metron_directory }}/bin/zk_load_configs.sh --mode PUSH -i {{ zookeeper_config_path }} -z {{ zookeeper_url }} && touch {{ zookeeper_config_path }}/configured"
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml b/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml
deleted file mode 100644
index a1aa237..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/threat_intel.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Create Bulk load working Directory
- file:
- path: "{{ threat_intel_work_dir }}"
- state: directory
-
-- name: Copy extractor.json to {{ inventory_hostname }}
- copy:
- src: ../roles/metron_streaming/files/extractor.json
- dest: "{{ threat_intel_work_dir }}"
- mode: 0644
-
-- name: Copy Bulk Load CSV File
- template:
- src: "{{ threat_intel_csv_filepath }}"
- dest: "{{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }}"
- mode: 0644
-
-- name: Copy Bulk Load CSV File to HDFS
- command: "hdfs dfs -put -f {{ threat_intel_work_dir }}/{{ threat_intel_csv_filename }} ."
-
-- name: Run Threat Intel Bulk Load
- shell: "{{ threat_intel_bin }} -c t -t {{threatintel_hbase_table}} -e {{ threat_intel_work_dir }}/extractor.json -i /user/root -m MR && touch {{ threat_intel_work_dir }}/loaded"
- args:
- creates: "{{ threat_intel_work_dir }}/loaded"
-
-- name: Clean up HDFS File
- command: "hdfs dfs -rm {{ threat_intel_csv_filename }}"
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/tasks/topologies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/tasks/topologies.yml b/metron-deployment/roles/metron_streaming/tasks/topologies.yml
deleted file mode 100644
index 827e861..0000000
--- a/metron-deployment/roles/metron_streaming/tasks/topologies.yml
+++ /dev/null
@@ -1,86 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Configure Metron Pcap Topology
- lineinfile: >
- dest={{ metron_pcap_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.pcap.out=", line: "kafka.pcap.out={{ pcap_hdfs_path }}" }
- - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
-
-- name: Configure Metron Enrichment topology
- lineinfile: >
- dest={{ metron_enrichment_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
- - { regexp: "threat.intel.tracker.table=", line: "threat.intel.tracker.table={{ tracker_hbase_table }}" }
- - { regexp: "threat.intel.tracker.cf=", line: "threat.intel.tracker.cf=t" }
- - { regexp: "threat.intel.simple.hbase.table=", line: "threat.intel.simple.hbase.table={{ threatintel_hbase_table }}" }
- - { regexp: "threat.intel.simple.hbase.cf=", line: "threat.intel.simple.hbase.cf=t" }
- - { regexp: "enrichment.simple.hbase.table=", line: "enrichment.simple.hbase.table={{ enrichment_hbase_table }}" }
- - { regexp: "enrichment.simple.hbase.cf=", line: "enrichment.simple.hbase.cf=t" }
-
-
-- name: Configure Metron Solr topology
- lineinfile: >
- dest={{ metron_solr_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/indexing/indexed" }
- - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
- - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
-
-- name: Configure Metron Elasticsearch topology
- lineinfile: >
- dest={{ metron_elasticsearch_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
- - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
- - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
- - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
- - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
- - { regexp: "index.hdfs.output=", line: "index.hdfs.output={{ metron_hdfs_output_dir }}/indexing/indexed" }
- - { regexp: "bolt.hdfs.rotation.policy=", line: "bolt.hdfs.rotation.policy={{ metron_hdfs_rotation_policy }}" }
- - { regexp: "bolt.hdfs.rotation.policy.count=", line: "bolt.hdfs.rotation.policy.count={{ metron_hdfs_rotation_policy_count}}" }
- - { regexp: "bolt.hdfs.rotation.policy.units=", line: "bolt.hdfs.rotation.policy.units={{ metron_hdfs_rotation_policy_units }}" }
-
-- name: Configure Profiler topology
- lineinfile: >
- dest={{ metron_profiler_properties_config_path }}
- regexp="{{ item.regexp }}"
- line="{{ item.line }}"
- with_items:
- - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
- - { regexp: "kafka.broker=", line: "kafka.broker={{ kafka_broker_url }}" }
-
-
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json b/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
deleted file mode 100644
index 87af1c0..0000000
--- a/metron-deployment/roles/metron_streaming/templates/config/elasticsearch.global.json
+++ /dev/null
@@ -1,7 +0,0 @@
-{
- "es.clustername": "{{ elasticsearch_cluster_name }}",
- "es.ip": "{{ groups.search[0] }}",
- "es.port": "{{ elasticsearch_transport_port }}",
- "es.date.format": "yyyy.MM.dd.HH",
- "parser.error.topic": "indexing"
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/templates/config/solr.global.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/templates/config/solr.global.json b/metron-deployment/roles/metron_streaming/templates/config/solr.global.json
deleted file mode 100644
index 5cb7a4d..0000000
--- a/metron-deployment/roles/metron_streaming/templates/config/solr.global.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
- "solr.zookeeper": "{{ zookeeper_url }}",
- "solr.collection": "{{ solr_collection_name }}",
- "solr.numShards": {{ solr_number_shards }},
- "solr.replicationFactor": {{ solr_replication_factor }}
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/templates/threat_ip.csv
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/templates/threat_ip.csv b/metron-deployment/roles/metron_streaming/templates/threat_ip.csv
deleted file mode 100644
index 3ac38f3..0000000
--- a/metron-deployment/roles/metron_streaming/templates/threat_ip.csv
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#Add single column of ip address to alert
-#Public lists are available on the internet
-# example:
-23.113.113.105
-24.107.205.249
-24.108.62.255
-24.224.153.71
-27.4.1.212
-27.131.149.102
-31.24.30.31
-31.131.251.33
-31.186.99.250
-31.192.209.119
-31.192.209.150
-31.200.244.17
-37.34.52.185
-37.58.112.101
-37.99.146.27
-37.128.132.96
-37.140.195.177
-37.140.199.100
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit-start/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit-start/defaults/main.yml b/metron-deployment/roles/monit-start/defaults/main.yml
deleted file mode 100644
index 26a05b3..0000000
--- a/metron-deployment/roles/monit-start/defaults/main.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-#
-# defines which services will be started. by default, no services
-# are started
-#
-services_to_start: []
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit-start/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit-start/tasks/main.yml b/metron-deployment/roles/monit-start/tasks/main.yml
deleted file mode 100644
index 68bf07a..0000000
--- a/metron-deployment/roles/monit-start/tasks/main.yml
+++ /dev/null
@@ -1,53 +0,0 @@
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Start monit
- service:
- name: monit
- state: started
- enabled: true
-
-- name: Reload monit definitions
- shell: monit reload
-
-- name: Stop all services
- shell: monit stop all
- register: result
- until: result.rc == 0
- retries: 20
- delay: 10
-
-- name: Find the services installed on the host
- shell: monit summary | tail -n +3 | awk -F"'" '{print $2}'
- register: installed_services
-
-# which services that need started are actually installed on this host?
-- set_fact:
- installed_services_to_start: "{{ services_to_start | intersect(installed_services.stdout_lines) }}"
-
-- name: Start Metron services
- debug:
- msg: "Attemping to start: {{ installed_services_to_start }}"
-
-- name: Wait for metron services to start
- shell: "monit start {{ item }}"
- with_items: "{{ installed_services_to_start }}"
- register: result
- until: result.rc == 0
- retries: 20
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/README.md b/metron-deployment/roles/monit/README.md
deleted file mode 100644
index 6f50852..0000000
--- a/metron-deployment/roles/monit/README.md
+++ /dev/null
@@ -1,79 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# Monit Integration
-
-This role will leverage Monit as a process watchdog to manage sensors, topologies, and core services.
-
-* Monit can be used to start, stop, or check status of any of the sensors or topologies.
-* When monitoring is enabled (on by default) if a process dies, it will be restarted.
-* The Monit web interface is exposed at http://hostname:2812.
-* The web interface username and password is defined by the `monit_user` and `monit_password` variables. These default to `admin`/`monit`.
-* Monit CLI tools can also be used to simplify the process of managing Metron components.
-* The post-deployment report for Amazon-EC2 provides links to Monit's web interface labeled as 'Sensor Status' and 'Topology Status.'
-
- ```
- ok: [localhost] => {
- "Success": [
- "Apache Metron deployed successfully",
- " Metron @ http://ec2-52-39-143-62.us-west-2.compute.amazonaws.com:5000",
- " Ambari @ http://ec2-52-39-4-93.us-west-2.compute.amazonaws.com:8080",
- " Sensor Status @ http://ec2-52-39-4-93.us-west-2.compute.amazonaws.com:2812",
- " Topology Status @ http://ec2-52-39-130-62.us-west-2.compute.amazonaws.com:2812",
- "For additional information, see https://metron.apache.org/'"
- ]
- }
- ```
-
-## Usage
-
-
-Start all Metron components
-
-```
-monit start all
-```
-
-Stop all Metron components
-
-```
-monit stop all
-```
-
-Start an individual Metron component
-
-```
-monit start bro-parser
-```
-
-Start all components required to ingest Bro data
-
-```
-monit -g bro start
-```
-
-Start all parsers
-
-```
-monit -g parsers start
-```
-
-What is running?
-
-```
-monit summary
-```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/defaults/main.yml b/metron-deployment/roles/monit/defaults/main.yml
deleted file mode 100644
index 651aa58..0000000
--- a/metron-deployment/roles/monit/defaults/main.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-monit_home: /usr/local/monit
-monit_config_home: /etc/monit.d
-monit_user: admin
-monit_pass: monit
-
-bro_pid_file: /usr/local/bro/spool/bro/.pid
-snort_alert_csv_path: /var/log/snort/alert.csv
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/main.yml b/metron-deployment/roles/monit/tasks/main.yml
deleted file mode 100644
index 9c1d75d..0000000
--- a/metron-deployment/roles/monit/tasks/main.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: monit.yml
-- include: monit-definitions.yml
-
-- include: monit-sensor-definitions.yml
- tags:
- - sensors
-
-- include: monit-stub-definitions.yml
- tags:
- - sensor-stubs
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-definitions.yml b/metron-deployment/roles/monit/tasks/monit-definitions.yml
deleted file mode 100644
index 13e00fc..0000000
--- a/metron-deployment/roles/monit/tasks/monit-definitions.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create monit definition for pcap-replay
- template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit
- when: ("sensors" in group_names) and (install_pcap_replay | default(False))
- tags: sensors
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml b/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml
deleted file mode 100644
index 4b2d5de..0000000
--- a/metron-deployment/roles/monit/tasks/monit-sensor-definitions.yml
+++ /dev/null
@@ -1,38 +0,0 @@
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create monit definition for pcap-replay
- template: src=monit/pcap-replay.monit dest={{ monit_config_home }}/pcap-replay.monit
- when: ("sensors" in group_names) and (install_pcap_replay | default(False))
-
-- name: Create monit definition for pycapa
- template: src=monit/pycapa.monit dest={{ monit_config_home }}/pycapa.monit
- when: ("sensors" in group_names) and (install_pycapa | default(True))
-
-- name: Create monit definition for snort
- template: src=monit/snort.monit dest={{ monit_config_home }}/snort.monit
- when: ("sensors" in group_names) and (install_snort | default(True))
-
-- name: Create monit definition for yaf
- template: src=monit/yaf.monit dest={{ monit_config_home }}/yaf.monit
- when: ("sensors" in group_names) and (install_yaf | default(True))
-
-- name: Create monit definition for bro
- template: src=monit/bro.monit dest={{ monit_config_home }}/bro.monit
- when: ("sensors" in group_names) and (install_bro | default(True))
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml b/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml
deleted file mode 100644
index fde711a..0000000
--- a/metron-deployment/roles/monit/tasks/monit-stub-definitions.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create monit definition for snort
- template: src=monit/snort-stub.monit dest={{ monit_config_home }}/snort.monit
- when: ("sensors" in group_names) and (install_snort | default(True))
-
-- name: Create monit definition for yaf
- template: src=monit/yaf-stub.monit dest={{ monit_config_home }}/yaf.monit
- when: ("sensors" in group_names) and (install_yaf | default(True))
-
-- name: Create monit definition for bro
- template: src=monit/bro-stub.monit dest={{ monit_config_home }}/bro.monit
- when: ("sensors" in group_names) and (install_bro | default(True))
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/tasks/monit.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/tasks/monit.yml b/metron-deployment/roles/monit/tasks/monit.yml
deleted file mode 100644
index bf300f4..0000000
--- a/metron-deployment/roles/monit/tasks/monit.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install monit
- yum:
- name: monit
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Deploy monit configuration
- template: src=monit/monit.conf dest=/etc/monit.conf
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/bro-stub.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/bro-stub.monit b/metron-deployment/roles/monit/templates/monit/bro-stub.monit
deleted file mode 100644
index 54bdbd6..0000000
--- a/metron-deployment/roles/monit/templates/monit/bro-stub.monit
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-check process bro
- with pidfile /var/run/sensor-stubs-bro.pid
- start program = "/etc/init.d/sensor-stubs start bro"
- stop program = "/etc/init.d/sensor-stubs stop bro"
- if does not exist then restart
- group bro
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/bro.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/bro.monit b/metron-deployment/roles/monit/templates/monit/bro.monit
deleted file mode 100644
index db6a318..0000000
--- a/metron-deployment/roles/monit/templates/monit/bro.monit
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process bro
- with pidfile {{ bro_pid_file }}
- start program = "/usr/local/bro/bin/broctl deploy"
- stop program = "/usr/local/bro/bin/broctl stop"
- restart program = "/usr/local/bro/bin/broctl restart"
- if does not exist then restart
- group bro
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/monit.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/monit.conf b/metron-deployment/roles/monit/templates/monit/monit.conf
deleted file mode 100644
index 660c421..0000000
--- a/metron-deployment/roles/monit/templates/monit/monit.conf
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-set logfile syslog
-set pidfile /var/run/monit.pid
-
-# check services at X second intervals
-set daemon 30
-include /etc/monit.d/*
-
-# allow http access
-set httpd port 2812 and
- use address {{ inventory_hostname }}
- allow 0.0.0.0/0
- allow {{ monit_user }}:{{ monit_pass }}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/pcap-replay.monit b/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
deleted file mode 100644
index da0c006..0000000
--- a/metron-deployment/roles/monit/templates/monit/pcap-replay.monit
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process pcap-replay
- with pidfile /var/run/pcap-replay.pid
- start program = "/etc/init.d/pcap-replay start"
- stop program = "/etc/init.d/pcap-replay stop"
- if does not exist then restart
- group pcap
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/pycapa.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/pycapa.monit b/metron-deployment/roles/monit/templates/monit/pycapa.monit
deleted file mode 100644
index 1123d87..0000000
--- a/metron-deployment/roles/monit/templates/monit/pycapa.monit
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process pycapa
- with pidfile /var/run/pycapa.pid
- start program = "/etc/init.d/pycapa start"
- stop program = "/etc/init.d/pycapa stop"
- if does not exist then restart
- group pcap
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/snort-stub.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/snort-stub.monit b/metron-deployment/roles/monit/templates/monit/snort-stub.monit
deleted file mode 100644
index b782690..0000000
--- a/metron-deployment/roles/monit/templates/monit/snort-stub.monit
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-check process snort
- with pidfile /var/run/sensor-stubs-snort.pid
- start program = "/etc/init.d/sensor-stubs start snort"
- stop program = "/etc/init.d/sensor-stubs stop snort"
- if does not exist then restart
- group snort
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/snort.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/snort.monit b/metron-deployment/roles/monit/templates/monit/snort.monit
deleted file mode 100644
index 6fb429b..0000000
--- a/metron-deployment/roles/monit/templates/monit/snort.monit
+++ /dev/null
@@ -1,33 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process snort matching "/usr/sbin/snort"
- start program = "/etc/init.d/snortd start"
- stop program = "/etc/init.d/snortd stop"
- if does not exist then restart
- group snort
- group sensors
- group metron
-
-check process snort-producer
- with pidfile /var/run/snort-producer.pid
- start program = "/etc/init.d/snort-producer start"
- stop program = "/etc/init.d/snort-producer stop"
- if does not exist then restart
- depends on snort
- group snort
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/yaf-stub.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/yaf-stub.monit b/metron-deployment/roles/monit/templates/monit/yaf-stub.monit
deleted file mode 100644
index 2a92a53..0000000
--- a/metron-deployment/roles/monit/templates/monit/yaf-stub.monit
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-check process yaf
- with pidfile /var/run/sensor-stubs-yaf.pid
- start program = "/etc/init.d/sensor-stubs start yaf"
- stop program = "/etc/init.d/sensor-stubs stop yaf"
- if does not exist then restart
- group yaf
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/monit/templates/monit/yaf.monit
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/monit/templates/monit/yaf.monit b/metron-deployment/roles/monit/templates/monit/yaf.monit
deleted file mode 100644
index 1f7b4d2..0000000
--- a/metron-deployment/roles/monit/templates/monit/yaf.monit
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-check process yaf
- with pidfile /var/run/yaf.pid
- start program = "/etc/init.d/yaf start"
- stop program = "/etc/init.d/yaf stop"
- if does not exist then restart
- group yaf
- group sensors
- group metron
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/ntp/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/ntp/tasks/main.yml b/metron-deployment/roles/ntp/tasks/main.yml
deleted file mode 100644
index 7b1b9a8..0000000
--- a/metron-deployment/roles/ntp/tasks/main.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install ntp
- yum:
- name: ntp
- state: present
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
-
-- name: Ensure ntp is running and enabled
- service:
- name: ntpd
- state: started
- enabled: yes
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/README.md b/metron-deployment/roles/opentaxii/README.md
deleted file mode 100644
index 7c111de..0000000
--- a/metron-deployment/roles/opentaxii/README.md
+++ /dev/null
@@ -1,178 +0,0 @@
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
--->
-# OpenTAXII
-
-Installs [OpenTAXII](https://github.com/EclecticIQ/OpenTAXII) as a deamon that can be launched via a SysV service script. The complementary client implementation, [Cabby](https://github.com/EclecticIQ/cabby) is also installed.
-
-OpenTAXII is a robust Python implementation of TAXII Services that delivers a rich feature set and friendly pythonic API. [TAXII](https://stixproject.github.io/) (Trusted Automated eXchange of Indicator Information) is a collection of specifications defining a set of services and message exchanges used for sharing cyber threat intelligence information between parties.
-
-## Getting Started
-
-After deployment completes the OpenTAXII service is installed and running. A set of [Hail a TAXII](http://hailataxii.com/) threat intel collections have been defined and configured. Use the `status` option to view the collections that have been defined.
-
-```
-$ service opentaxii status
-Checking opentaxii... Running
-guest.phishtank_com 0
-guest.Abuse_ch 0
-guest.CyberCrime_Tracker 0
-guest.EmergingThreats_rules 0
-guest.Lehigh_edu 0
-guest.MalwareDomainList_Hostlist 0
-guest.blutmagie_de_torExits 0
-guest.dataForLast_7daysOnly 0
-guest.dshield_BlockList 0
-```
-
-Notice that each collections contain zero records. None of the data is automatically synced during deployment. To sync the data manually use the `sync` option as defined below. The following example does not provide a begin and end time so the data will be fetched for the current day only.
-
-```
-# service opentaxii sync guest.blutmagie_de_torExits
-2016-04-21 20:34:42,511 INFO: Starting new HTTP connection (1): localhost
-2016-04-21 20:34:42,540 INFO: Response received for Inbox_Message from http://localhost:9000/services/inbox
-2016-04-21 20:34:42,542 INFO: Sending Inbox_Message to http://localhost:9000/services/inbox
-...
-2016-04-21 20:34:42,719 INFO: Response received for Poll_Request from http://localhost:9000/services/poll
-2016-04-21 20:34:42,719 INFO: Content blocks count: 1618, is partial: False
-```
-
-The OpenTAXII service now contains 1,618 threat intel records indicating Tor Exit nodes.
-
-```
-[root@source ~]# service opentaxii status
-Checking opentaxii... Running
-guest.phishtank_com 0
-guest.Abuse_ch 0
-guest.CyberCrime_Tracker 0
-guest.EmergingThreats_rules 0
-guest.Lehigh_edu 0
-guest.MalwareDomainList_Hostlist 0
-guest.blutmagie_de_torExits 1618
-guest.dataForLast_7daysOnly 0
-guest.dshield_BlockList 0
-```
-
-## Usage
-
-A standard SysV script has been installed to manage OpenTAXII. The following functions are available.
-
-`start` `stop` `restart` the OpenTAXII service
-
-`status` of the OpenTAXII service. The command displays the collections that have been defined and the number of records in each.
-
-```
-$ service opentaxii status
-Checking opentaxii... Running
-guest.phishtank_com 984
-guest.Abuse_ch 45
-guest.CyberCrime_Tracker 482
-guest.EmergingThreats_rules 0
-guest.Lehigh_edu 1030
-guest.MalwareDomainList_Hostlist 84
-guest.blutmagie_de_torExits 3236
-guest.dataForLast_7daysOnly 3377
-guest.dshield_BlockList 0
-```
-
-`setup` Initializes the services and collections required to operate the OpenTAXII service. This will destroy all existing data. The user is prompted to continue before any data is destroyed.
-
-```
-# service opentaxii setup
-WARNING: force reset and destroy all opentaxii data? [Ny]: y
-Stopping opentaxii ..Ok
-2016-04-21T19:56:01.886157Z [opentaxii.server] info: api.persistence.loaded {timestamp=2016-04-21T19:56:01.886157Z, logger=opentaxii.server, api_class=SQLDatabaseAPI, event=api.persistence.loaded, level=info}
-2016-04-21T19:56:01.896503Z [opentaxii.server] info: api.auth.loaded {timestamp=2016-04-21T19:56:01.896503Z, logger=opentaxii.server, api_class=SQLDatabaseAPI, event=api.auth.loaded, level=info}
-2016-04-21T19:56:01.896655Z [opentaxii.server] info: taxiiserver.configured {timestamp=2016-04-21T19:56:01.896655Z, logger=opentaxii.server, event=taxiiserver.configured, level=info}
-...
-Ok
-```
-
-`sync [collection] [begin-at] [end-at]` Syncs the threat intel data available at [Hail a TAXII](http://hailataxii.com/). If no begin and end date is provided then data is synced over the current day only.
- - `collection` Name of the collection to sync.
- - `begin-at` Exclusive begin of time window; ISO8601
- - `end-at` Inclusive end of time window; ISO8601
-
-```
-$ service opentaxii sync guest.phishtank_com
-+ /usr/local/opentaxii/opentaxii-venv/bin/taxii-proxy --poll-path http://hailataxii.com/taxii-data --poll-collection guest.phishtank_com --inbox-path http://localhost:9000/services/guest.phishtank_com-inbox --inbox-collection guest.phishtank_com --binding urn:stix.mitre.org:xml:1.1.1 --begin 2016-04-21 --end 2016-04-22
-2016-04-21 17:36:23,778 INFO: Sending Poll_Request to http://hailataxii.com/taxii-data
-2016-04-21 17:36:23,784 INFO: Starting new HTTP connection (1): hailataxii.com
-2016-04-21 17:36:24,175 INFO: Response received for Poll_Request from http://hailataxii.com/taxii-data
-2016-04-21 17:36:24,274 INFO: Sending Inbox_Message to http://localhost:9000/services/guest.phishtank_com-inbox
-...
-2016-04-21 17:36:34,867 INFO: Response received for Poll_Request from http://localhost:9000/services/guest.phishtank_com-poll
-2016-04-21 17:36:34,868 INFO: Content blocks count: 6993, is partial: False
-```
-
-### Troubleshooting
-
-Should you need to explore the installation, here are instructions on doing so.
-
-OpenTAXII is installed in a virtual environment. Before exploring the environment run the following commands to perform the necessary setup. The specific paths may change depending on your Ansible settings.
-
-```
-export LD_LIBRARY_PATH=/opt/rh/python27/root/usr/lib64
-export OPENTAXII_CONFIG=/usr/local/opentaxii/etc/opentaxii-conf.yml
-cd /usr/local/opentaxii
-. opentaxii-venv/bin/activate
-```
-
-Discover available services.
-
-```
-taxii-discovery --discovery http://localhost:9000/services/discovery
-taxii-discovery --discovery http://hailataxii.com/taxii-data
-```
-
-Explore available collections.
-
-```
-taxii-collections --discovery http://localhost:9000/services/discovery
-taxii-collections --discovery http://hailataxii.com/taxii-data
-```
-
-Read data from a collection.
-
-```
-taxii-poll --discovery http://localhost:9000/services/discovery -c guest.phishtank_com
-taxii-poll --discovery http://hailataxii.com/taxii-data -c guest.phishtank_com --begin 2016-04-20
-```
-
-Manually load data into a collection.
-
-```
-taxii-push \
- --discovery http://localhost:9000/services/discovery \
- --dest phishtank \
- --content-file data.xml \
- --username guest \
- --password guest
-```
-
-Fetch data from a remote service and mirror it locally.
-
-```
-taxii-proxy --poll-path http://hailataxii.com/taxii-data \
- --poll-collection guest.phishtank_com \
- --inbox-path http://localhost:9000/services/guest.phishtank_com-inbox \
- --inbox-collection guest.phishtank_com \
- --binding urn:stix.mitre.org:xml:1.1.1 \
- --inbox-username guest \
- --inbox-password guest \
- --begin 2016-04-20
-```
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/defaults/main.yml b/metron-deployment/roles/opentaxii/defaults/main.yml
deleted file mode 100644
index 9ab86cb..0000000
--- a/metron-deployment/roles/opentaxii/defaults/main.yml
+++ /dev/null
@@ -1,43 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-opentaxii_host: localhost
-opentaxii_port: 9000
-opentaxii_domain: "http://{{ opentaxii_host }}:{{ opentaxii_port }}"
-opentaxii_bind: "{{ opentaxii_host }}:{{ opentaxii_port }}"
-opentaxii_home: /usr/local/opentaxii
-opentaxii_venv: opentaxii-venv
-opentaxii_bin: "{{ opentaxii_home }}/{{ opentaxii_venv }}/bin"
-opentaxii_user: guest
-opentaxii_pass: guest
-opentaxii_workers: 2
-opentaxii_loglevel: info
-opentaxii_timeout: 300
-opentaxii_auth_db: "{{ opentaxii_home }}/data/auth.db"
-opentaxii_data_db: "{{ opentaxii_home }}/data/data.db"
-opentaxii_salt: "@#L:KJDASLKJASD@"
-python27_home: /opt/rh/python27/root
-opentaxii_available_collections:
- - guest.phishtank_com
- - guest.Abuse_ch
- - guest.CyberCrime_Tracker
- - guest.EmergingThreats_rules
- - guest.Lehigh_edu
- - guest.MalwareDomainList_Hostlist
- - guest.blutmagie_de_torExits
- - guest.dataForLast_7daysOnly
- - guest.dshield_BlockList
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/meta/main.yml b/metron-deployment/roles/opentaxii/meta/main.yml
deleted file mode 100644
index 841d185..0000000
--- a/metron-deployment/roles/opentaxii/meta/main.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/tasks/dependencies.yml b/metron-deployment/roles/opentaxii/tasks/dependencies.yml
deleted file mode 100644
index 3b2b38a..0000000
--- a/metron-deployment/roles/opentaxii/tasks/dependencies.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install yum repositories
- yum: name={{ item }} update_cache=yes
- with_items:
- - epel-release
- - centos-release-scl
-
-- name: Install dependencies
- yum: name={{ item }}
- with_items:
- - "@Development tools"
- - python27
- - python27-scldevel
- - python27-python-virtualenv
- - libxml2-devel
- - libxslt-devel
- - libselinux-python
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/hailataxii.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/tasks/hailataxii.yml b/metron-deployment/roles/opentaxii/tasks/hailataxii.yml
deleted file mode 100644
index 1eebfe6..0000000
--- a/metron-deployment/roles/opentaxii/tasks/hailataxii.yml
+++ /dev/null
@@ -1,45 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Deploy service/collection definitions for hailataxii.com
- template: src={{ item }} dest={{ opentaxii_home }}/etc mode=0400
- with_items:
- - services.yml
- - collections.yml
-
-- name: Add collection definitions for hailataxii.com
- blockinfile:
- dest: "{{ opentaxii_home }}/etc/collections.yml"
- marker: "# {mark} ANSIBLE MANAGED BLOCK {{ item }}"
- block: |
- - name: {{ item }}
- type: DATA_SET
- available: true
- accept_all_content: true
- supported_content:
- - urn:stix.mitre.org:xml:1.1.1
- service_ids:
- - inbox
- - collection
- - poll
- with_items: "{{ opentaxii_available_collections }}"
-
-- name: Setup opentaxii
- shell: /etc/init.d/opentaxii setup
-
-- name: Start opentaxii
- service: name=opentaxii state=restarted
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/tasks/main.yml b/metron-deployment/roles/opentaxii/tasks/main.yml
deleted file mode 100644
index baa6b35..0000000
--- a/metron-deployment/roles/opentaxii/tasks/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: opentaxii.yml
-- include: hailataxii.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/tasks/opentaxii.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/tasks/opentaxii.yml b/metron-deployment/roles/opentaxii/tasks/opentaxii.yml
deleted file mode 100644
index c153149..0000000
--- a/metron-deployment/roles/opentaxii/tasks/opentaxii.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create install directory
- file: path={{ item }} state=directory mode=0755
- with_items:
- - "{{ opentaxii_home }}"
- - "{{ opentaxii_home }}/etc"
- - "{{ opentaxii_home }}/data"
- - "{{ opentaxii_home }}/bin"
-
-- name: Create virtual environment
- shell: "{{ python27_home }}/usr/bin/virtualenv {{ opentaxii_venv }}"
- args:
- chdir: "{{ opentaxii_home }}"
- creates: "{{ opentaxii_home }}/{{ opentaxii_venv }}"
- environment:
- LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
-
-- name: Install opentaxii, cabby, and gunicorn
- shell: "{{ opentaxii_bin }}/pip install {{ item }}"
- environment:
- LD_LIBRARY_PATH: "{{ python27_home }}/usr/lib64"
- with_items:
- - opentaxii
- - cabby
- - gunicorn
-
-- name: Deploy collection status script
- template: src=collection-status.py dest={{ opentaxii_home }}/bin mode=0755
-
-- name: Deploy opentaxii configs
- template: src=opentaxii-conf.yml dest={{ opentaxii_home }}/etc mode=0400
-
-- name: Deploy opentaxii service script
- template: src=opentaxii dest=/etc/init.d/opentaxii mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/collection-status.py
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/collection-status.py b/metron-deployment/roles/opentaxii/templates/collection-status.py
deleted file mode 100644
index 2d912c9..0000000
--- a/metron-deployment/roles/opentaxii/templates/collection-status.py
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-from cabby import create_client
-
-try:
- # create a connection
- client = create_client(host='{{ opentaxii_host }}', port='{{ opentaxii_port }}', discovery_path='/services/discovery')
-
- # iterate through each defined collection
- collections = client.get_collections(uri='{{ opentaxii_domain }}/services/collection')
-
- for collection in collections:
- # how many records in each collection?
- count = client.get_content_count(collection_name=collection.name, uri='{{ opentaxii_domain }}/services/poll')
- print "%-50s %-10d" % (collection.name, count.count)
-except:
- print "Services not defined"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/collections.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/collections.yml b/metron-deployment/roles/opentaxii/templates/collections.yml
deleted file mode 100644
index 07a0e5a..0000000
--- a/metron-deployment/roles/opentaxii/templates/collections.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-collections:
-# intentionally blank - managed by ansible
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/opentaxii/templates/opentaxii
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/opentaxii/templates/opentaxii b/metron-deployment/roles/opentaxii/templates/opentaxii
deleted file mode 100644
index e934e7f..0000000
--- a/metron-deployment/roles/opentaxii/templates/opentaxii
+++ /dev/null
@@ -1,176 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# opentaxii daemon
-# chkconfig: 345 20 80
-# description: OpenTAXII is a robust Python implementation of TAXII Service
-# processname: opentaxii
-#
-NAME=opentaxii
-DESC="OpenTAXII is a robust Python implementation of a TAXII service"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-LOGFILE="/var/log/$NAME.log"
-EXTRA_ARGS="${@:2}"
-CONFIRM_TIMEOUT=3
-DAEMON_PATH="{{ opentaxii_home }}"
-
-export LD_LIBRARY_PATH={{ python27_home }}/usr/lib64
-export OPENTAXII_CONFIG={{ opentaxii_home }}/etc/opentaxii-conf.yml
-
-case "$1" in
-
- ##############################################################################
- # start
- #
- start)
- printf "%-50s" "Starting $NAME..."
-
- # setup virtual environment
- cd $DAEMON_PATH
- . {{ opentaxii_bin }}/activate
-
- # kick-off the daemon
- DAEMON="{{ opentaxii_bin }}/gunicorn"
- DAEMONOPTS="opentaxii.http:app"
- DAEMONOPTS+=" --daemon"
- DAEMONOPTS+=" --pid $PIDFILE"
- DAEMONOPTS+=" --workers {{ opentaxii_workers }}"
- DAEMONOPTS+=" --log-level {{ opentaxii_loglevel }}"
- DAEMONOPTS+=" --log-file $LOGFILE"
- DAEMONOPTS+=" --timeout {{ opentaxii_timeout }}"
- DAEMONOPTS+=" --bind {{ opentaxii_bind }}"
- DAEMONOPTS+=" --env OPENTAXII_CONFIG={{ opentaxii_home }}/etc/opentaxii-conf.yml"
- DAEMONOPTS+=" $EXTRA_ARGS"
- PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1`
- printf "%s\n" "Ok"
- ;;
-
- ##############################################################################
- # status
- #
- status)
- printf "%-50s" "Checking $NAME..."
- . {{ opentaxii_bin }}/activate
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- printf "%s\n" "Running"
- {{ opentaxii_home }}/bin/collection-status.py
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- ##############################################################################
- # stop
- #
- stop)
- printf "%-50s" "Stopping $NAME"
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- while sleep 1
- echo -n "."
- kill -0 $PID >/dev/null 2>&1
- do
- kill $PID
- done
-
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- ##############################################################################
- # restart
- #
- restart)
- $0 stop
- $0 start
- ;;
-
- ##############################################################################
- # setup
- #
- setup)
-
- # if the database file already exists; prompt for confirmation
- if [ -f "{{ opentaxii_data_db }}" ]; then
- read -t $CONFIRM_TIMEOUT -p "WARNING: force reset and destroy all opentaxii data? [Ny]: " REPLY
- if [[ ! $REPLY =~ ^[Yy]$ ]]; then
- exit 0
- fi
- fi
-
- $0 stop
- cd $DAEMON_PATH
- rm -f {{ opentaxii_auth_db }}
- rm -f {{ opentaxii_data_db }}
- {{ opentaxii_bin }}/opentaxii-create-account --username {{ opentaxii_user }} --password {{ opentaxii_pass }}
- {{ opentaxii_bin }}/opentaxii-create-services -c {{ opentaxii_home}}/etc/services.yml
- {{ opentaxii_bin }}/opentaxii-create-collections -c {{ opentaxii_home}}/etc/collections.yml
- printf "%s\n" "Ok"
- ;;
-
- ##############################################################################
- # sync
- #
- sync)
-
- # collect the arguments
- POLL_SOURCE="http://hailataxii.com/taxii-data"
- COLL="$2"
- BEGIN="${3:-`date --iso-8601`}"
- END="${4:-`date --date=tomorrow --iso-8601`}"
-
- # validation
- if [ -z "$COLL" ]; then
- echo "$0 sync [COLLECTION] [BEGIN-AT] [END-AT]"
- echo "error: missing name of collection"
- exit 1
- fi
-
- # sync the data
- set -x
- {{ opentaxii_bin }}/taxii-proxy \
- --poll-path $POLL_SOURCE \
- --poll-collection $COLL \
- --inbox-path {{ opentaxii_domain }}/services/inbox \
- --inbox-collection $COLL \
- --binding urn:stix.mitre.org:xml:1.1.1 \
- --begin $BEGIN \
- --end $END
- set +x
-
- # count the number of records in the local collection
- {{ opentaxii_bin }}/taxii-poll \
- --discovery {{ opentaxii_domain }}/services/discovery \
- --collection $COLL \
- --count-only
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart|setup|sync}"
- exit 1
-esac
[19/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/ansible/roles/yaf/templates/yaf
----------------------------------------------------------------------
diff --git a/metron-deployment/ansible/roles/yaf/templates/yaf b/metron-deployment/ansible/roles/yaf/templates/yaf
new file mode 100644
index 0000000..18bc4ac
--- /dev/null
+++ b/metron-deployment/ansible/roles/yaf/templates/yaf
@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# yaf daemon
+# chkconfig: 345 20 80
+# description: Runs yaf - yet another flowmeter
+# processname: yaf
+#
+NAME=yaf
+DESC="Executes yaf - yet another flowmeter"
+PIDFILE=/var/run/$NAME.pid
+SCRIPTNAME=/etc/init.d/$NAME
+DAEMON_PATH="{{ yaf_home }}"
+DAEMON="{{ yaf_start }}"
+DAEMONOPTS="${@:2}"
+
+case "$1" in
+ start)
+ printf "%-50s" "Starting $NAME..."
+
+ # kick-off the daemon
+ cd $DAEMON_PATH
+ PID=`$DAEMON $DAEMONOPTS > /dev/null 2>&1 & echo $!`
+ if [ -z $PID ]; then
+ printf "%s\n" "Fail"
+ else
+ echo $PID > $PIDFILE
+ printf "%s\n" "Ok"
+ fi
+ ;;
+
+ status)
+ printf "%-50s" "Checking $NAME..."
+ if [ -f $PIDFILE ]; then
+ PID=`cat $PIDFILE`
+ if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
+ printf "%s\n" "Process dead but pidfile exists"
+ else
+ echo "Running"
+ fi
+ else
+ printf "%s\n" "Service not running"
+ fi
+ ;;
+
+ stop)
+ printf "%-50s" "Stopping $NAME"
+ PID=`cat $PIDFILE`
+ cd $DAEMON_PATH
+ if [ -f $PIDFILE ]; then
+ kill -HUP $PID
+ killall $NAME
+ printf "%s\n" "Ok"
+ rm -f $PIDFILE
+ else
+ printf "%s\n" "pidfile not found"
+ fi
+ ;;
+
+ restart)
+ $0 stop
+ $0 start
+ ;;
+
+ *)
+ echo "Usage: $0 {status|start|stop|restart}"
+ exit 1
+esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/README.md b/metron-deployment/development/README.md
new file mode 100644
index 0000000..bc99809
--- /dev/null
+++ b/metron-deployment/development/README.md
@@ -0,0 +1,24 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+# Metron Development Environments
+
+This directory contains environments useful for Metron developers. These environments are not intended for proof-of-concept, testing, or production use. These are extremely resource constrained and cannot support anything beyond the most basic work loads.
+
+* Metron running on CentOS 6
+* Metron running on Ubuntu 14
+* Fastcapa
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/.gitignore b/metron-deployment/development/centos6/.gitignore
new file mode 100644
index 0000000..8000dd9
--- /dev/null
+++ b/metron-deployment/development/centos6/.gitignore
@@ -0,0 +1 @@
+.vagrant
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/README.md b/metron-deployment/development/centos6/README.md
new file mode 100644
index 0000000..5132c30
--- /dev/null
+++ b/metron-deployment/development/centos6/README.md
@@ -0,0 +1,105 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Metron on CentOS 6
+==================
+
+This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running CentOS 6.
+
+Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services.
+
+Getting Started
+---------------
+
+### Prerequisites
+
+The computer used to deploy Apache Metron will need to have the following components installed.
+
+ - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
+ - [Docker](https://www.docker.com/community-edition)
+ - [Vagrant](https://www.vagrantup.com) 1.8+
+ - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
+ - [Virtualbox](https://virtualbox.org) 5.0+
+ - Python 2.7
+ - Maven 3.3.9
+ - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11)
+
+Running the following script can help validate whether you have all the prerequisites installed and running correctly.
+
+ ```
+ metron-deployment/scripts/platform-info.sh
+ ```
+
+#### How do I install these on MacOS?
+
+Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project.
+
+1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/).
+
+1. Run the following command in a terminal to install all of the required tools.
+
+ ```
+ brew cask install vagrant virtualbox docker
+ brew cask install caskroom/versions/java8
+ brew install maven@3.3 git
+ pip install ansible==2.2.2.0
+ vagrant plugin install vagrant-hostmanager
+ open /Applications/Docker.app
+ ```
+
+### Deploy Metron
+
+1. Ensure that the Docker service is running.
+
+1. Deploy Metron
+
+ ```
+ cd metron-deployment/development/centos6
+ vagrant up
+ ```
+
+ Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.
+
+ ```
+ vagrant provision
+ ```
+
+### Explore Metron
+
+Navigate to the following resources to explore your newly minted Apache Metron environment.
+
+* [Metron Alerts](http://node1:4201)
+* [Ambari](http://node1:8080)
+
+Connecting to the host through SSH is as simple as running the following command.
+```
+vagrant ssh
+```
+
+### Working with Metron
+
+In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components.
+
+```
+vagrant --ansible-tags="sensor-stubs" provision
+```
+
+Tags are listed in the playbooks, some frequently used tags:
++ `hdp-install` - Install HDP
++ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
++ `sensors` - Deploy and start the sensors.
++ `sensor-stubs` - Deploy and start the sensor stubs.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/Vagrantfile b/metron-deployment/development/centos6/Vagrantfile
new file mode 100644
index 0000000..ed74b76
--- /dev/null
+++ b/metron-deployment/development/centos6/Vagrantfile
@@ -0,0 +1,92 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'getoptlong'
+
+ansibleTags=''
+ansibleSkipTags='sensors'
+
+begin
+ opts = GetoptLong.new(
+ [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ],
+ [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ]
+ )
+
+ opts.quiet = TRUE
+
+ opts.each do |opt, arg|
+ case opt
+ when '--ansible-tags'
+ ansibleTags=arg
+ when '--ansible-skip-tags'
+ ansibleSkipTags=arg
+ end
+ end
+rescue Exception => ignored
+#Ignore to allow other opts to be passed to Vagrant
+end
+
+puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != ''
+puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != ''
+
+hosts = [{
+ hostname: "node1",
+ ip: "192.168.66.121",
+ memory: "8192",
+ cpus: 4,
+ promisc: 2 # enables promisc on the 'Nth' network interface
+}]
+
+Vagrant.configure(2) do |config|
+
+ # all hosts built on centos 6
+ config.vm.box = "metron/centos_base"
+ config.ssh.insert_key = true
+
+ # enable the hostmanager plugin
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ # host definition
+ hosts.each_with_index do |host, index|
+ config.vm.define host[:hostname] do |node|
+
+ # host settings
+ node.vm.hostname = host[:hostname]
+ node.vm.network "private_network", ip: host[:ip]
+
+ # vm settings
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = host[:memory]
+ vb.cpus = host[:cpus]
+
+ # enable promisc mode on the network interface
+ if host.has_key?(:promisc)
+ vb.customize ["modifyvm", :id, "--nicpromisc#{host[:promisc]}", "allow-all"]
+ end
+ end
+ end
+ end
+
+ # provision the host with ansible
+ config.vm.provision :ansible do |ansible|
+ ansible.playbook = "../../ansible/playbooks/metron_full_install.yml"
+ ansible.sudo = true
+ ansible.tags = ansibleTags.split(",") if ansibleTags != ''
+ ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
+ ansible.inventory_path = "ansible/inventory"
+ end
+end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible.cfg b/metron-deployment/development/centos6/ansible.cfg
new file mode 100644
index 0000000..ade50ae
--- /dev/null
+++ b/metron-deployment/development/centos6/ansible.cfg
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[defaults]
+host_key_checking = false
+library = ../../ansible/extra_modules
+roles_path = ../../ansible/roles
+pipelining = True
+log_path = ./ansible.log
+
+
+# fix for "ssh throws 'unix domain socket too long' " problem
+[ssh_connection]
+control_path = %(directory)s/%%h-%%p-%%r
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/inventory/group_vars/all b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
new file mode 100644
index 0000000..347d82a
--- /dev/null
+++ b/metron-deployment/development/centos6/ansible/inventory/group_vars/all
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# only need to build the RPMs for CentOS
+metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-rpms"
+
+# ambari
+ambari_host: "{{ groups.ambari_master[0] }}"
+hdp_host_group: "{{ groups.ambari_slave }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
+cluster_type: single_node_vm
+ambari_server_mem: 512
+java_home: /usr/jdk64/jdk1.8.0_77
+
+# hbase
+pcap_hbase_table: pcap
+tracker_hbase_table: access_tracker
+threatintel_hbase_table: threatintel
+enrichment_hbase_table: enrichment
+
+# metron
+metron_version: 0.4.3
+metron_directory: /usr/metron/{{ metron_version }}
+bro_version: "2.5.2"
+fixbuf_version: "1.7.1"
+yaf_version: "2.8.0"
+daq_version: "2.0.6-1"
+pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+pycapa_home: "/opt/pycapa"
+snort_version: "2.9.8.0-1"
+snort_alert_csv_path: "/var/log/snort/alert.csv"
+threat_intel_bulk_load: False
+
+# data directories - only required to override defaults
+zookeeper_data_dir: "/data1/hadoop/zookeeper"
+namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
+namenode_name_dir: "/data1/hadoop/hdfs/namenode"
+datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
+journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
+nodemanager_local_dirs: "/data1/hadoop/yarn/local"
+timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
+timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
+nodemanager_log_dirs: "/data1/hadoop/yarn/log"
+jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
+storm_local_dir: "/data1/hadoop/storm"
+kafka_log_dirs: "/data1/kafka-log"
+elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
+
+# sensors
+sensor_test_mode: True
+install_pycapa: False
+install_bro: True
+install_snort: True
+install_yaf: False
+install_pcap_replay: True
+sniff_interface: eth1
+pcap_replay_interface: "{{ sniff_interface }}"
+pcapservice_port: 8081
+
+# search
+install_elasticsearch: True
+install_solr: False
+solr_collection_name: Metron
+solr_number_shards: 1
+solr_replication_factor: 1
+elasticsearch_transport_port: 9300
+## The elasticsearch_network_interface must be in the form specified for Elasticsearch, with leading and trailing underscores.
+elasticsearch_network_interface: _eth1_
+elasticsearch_web_port: 9200
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/centos6/ansible/inventory/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/development/centos6/ansible/inventory/hosts b/metron-deployment/development/centos6/ansible/inventory/hosts
new file mode 100644
index 0000000..9bd9ea1
--- /dev/null
+++ b/metron-deployment/development/centos6/ansible/inventory/hosts
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[ambari_master]
+node1
+
+[ambari_slave]
+node1
+
+[metron]
+node1
+
+[search]
+node1
+
+[sensors]
+node1
+
+[pcap_server]
+node1
+
+[web]
+node1
+
+[zeppelin]
+node1
+
+[monit:children]
+sensors
+pcap_server
+
+[local]
+127.0.0.1
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/.gitignore
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/.gitignore b/metron-deployment/development/fastcapa/.gitignore
new file mode 100644
index 0000000..a8b42eb
--- /dev/null
+++ b/metron-deployment/development/fastcapa/.gitignore
@@ -0,0 +1 @@
+*.retry
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/README.md b/metron-deployment/development/fastcapa/README.md
new file mode 100644
index 0000000..213d115
--- /dev/null
+++ b/metron-deployment/development/fastcapa/README.md
@@ -0,0 +1,138 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Fastcapa Test Environment
+=========================
+
+Provides a test environment for the development and testing of Fastcapa. The environment is automatically validated after it is created to ensure that Fastcapa is behaving correctly.
+
+Two virtualized nodes are launched with Vagrant that can communicate with one another over a private network.
+- The `source` node uses Metron's `pcap_replay` functionality to transmit raw network packet data over a private network.
+- The `sink` node is running `fastcapa` and is capturing these network packets.
+- Fastcapa then transforms and bundles the packets into a message.
+- The message is sent to a Kafka broker running on the `source` node.
+
+Getting Started
+---------------
+
+The Fastcapa test environment can be executed on different operating systems. There is a sub-directory for each operating system that Fastcapa can be tested on.
+
+To run, simply execute `vagrant up` within the appropriate directory. For example, to run the tests on CentOS 7.1 then execute the following commands.
+```
+cd centos-7.1
+vagrant up
+```
+
+Automated tests are executed after provisioning completes to ensure that Fastcapa and the rest of the environment is functioning properly. If you see something like the following, then the tests have passed.
+```
+$ vagrant up
+==> source: Running provisioner: ansible...
+ source: Running ansible-playbook...
+...
+TASK [debug] *******************************************************************
+ok: [source] => {
+ "msg": "Successfully received packets sent from pcap-replay!"
+}
+...
+TASK [debug] *******************************************************************
+ok: [source] => {
+ "msg": "Successfully received a Kafka message from fastcapa!"
+}
+```
+
+If the deployment process fails mid-course, running `vagrant provision` will continue the process from where it left off. This can sometimes occur when the VM reboots as part of the deployment process. The error might look like the following.
+```
+TASK [fastcapa : Restart for modified kernel params] ***************************
+fatal: [sink]: UNREACHABLE! => {"changed": false, "msg": "Failed to connect to the host via ssh: Shared connection to 127.0.0.1 closed.\r\n", "unreachable": true}
+ to retry, use: --limit @/Users/nallen/Development/metron/metron-deployment/vagrant/fastcapa-test-platform/playbook.retry
+
+PLAY RECAP *********************************************************************
+sink : ok=11 changed=9 unreachable=1 failed=0
+source : ok=29 changed=25 unreachable=0 failed=0
+
+Ansible failed to complete successfully. Any error output should be
+visible above. Please fix these errors and try again.
+```
+
+Going Deeper
+------------
+
+This section will outline in more detail the environment and how to interact with it.
+
+### `source`
+
+To validate that the `source` node is functioning properly, run the following commands.
+
+First, ensure that the `pcap-replay` service is running.
+
+```
+vagrant ssh source
+sudo service pcap-replay status
+```
+
+Use `tcpdump` to ensure that the raw packet data is being sent over the private network. Enter 'CTRL-C' to kill the `tcpdump` process once you are able to see that packets are being sent.
+
+```
+sudo yum -y install tcpdump
+sudo tcpdump -i enp0s8
+```
+
+### `sink`
+
+Next validate that the `sink` is functioning properly. Run the following commands starting from the host operating system.
+
+First, ensure that the `fastcapa` service is running.
+
+```
+vagrant ssh sink
+service fastcapa status
+```
+
+Ensure that the raw network packet data is being received by Kafka.
+
+```
+/usr/hdp/current/kafka-broker/bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic pcap
+```
+
+Enter 'CTRL-C' to kill the `kafka-console-consumer` process once you are able to see that packets are being sent. These packets will appear to be gibberish in the console. This is the raw binary network packet data after all.
+
+FAQ
+---
+
+### Error Message: Timed out while waiting for the machine to boot
+
+```
+Timed out while waiting for the machine to boot. This means that
+Vagrant was unable to communicate with the guest machine within
+the configured ("config.vm.boot_timeout" value) time period.
+If you look above, you should be able to see the error(s) that
+Vagrant had when attempting to connect to the machine. These errors
+are usually good hints as to what may be wrong.
+If you're using a custom box, make sure that networking is properly
+working and you're able to connect to the machine. It is a common
+problem that networking isn't setup properly in these boxes.
+Verify that authentication configurations are also setup properly,
+as well.
+If the box appears to be booting properly, you may want to increase
+the timeout ("config.vm.boot_timeout") value.
+➜ centos-7.4 git:(master) ✗ vagrant status
+Current machine states:
+source running (virtualbox)
+sink not created (virtualbox)
+```
+
+If you are unable to launch any of the Fastcapa test environments, which results in a message like the one above, then you may need to upgrade your version of Virtualbox. Success has been reported with versions of VirtualBox 5.1.22+.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile b/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
new file mode 100644
index 0000000..179ca34
--- /dev/null
+++ b/metron-deployment/development/fastcapa/centos-7.1/Vagrantfile
@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Vagrant.configure("2") do |config|
+
+ # enable hostmanager
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ #
+ # source
+ #
+ config.vm.define "source" do |node|
+
+ # host settings
+ node.vm.hostname = "source"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 1024
+ vb.cpus = 1
+ end
+ end
+
+ #
+ # sink
+ #
+ config.vm.define "sink" do |node|
+
+ # host settings
+ node.vm.hostname = "sink"
+ node.vm.box = "bento/centos-7.1"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 4096
+ vb.cpus = 3
+
+ # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
+ vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ end
+
+ # provision host
+ node.vm.provision :ansible do |ansible|
+ ansible.limit = "all"
+ ansible.playbook = "../playbook.yml"
+ ansible.extra_vars = "vars/main.yml"
+ end
+ end
+
+
+end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg b/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
new file mode 100644
index 0000000..dd78cfd
--- /dev/null
+++ b/metron-deployment/development/fastcapa/centos-7.1/ansible.cfg
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+host_key_checking = false
+library = ../../../ansible/extra_modules
+roles_path = ../../../ansible/roles
+pipelining = True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml b/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml
new file mode 100644
index 0000000..563fc2a
--- /dev/null
+++ b/metron-deployment/development/fastcapa/centos-7.1/vars/main.yml
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+# 0.9.4+ required for fastcapa
+librdkafka_version: 0.9.4
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+
+dpdk_device: ["00:08.0"]
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+kafka_broker_url: source:9092
+zookeeper_url: source:2181
+pcap_replay_interface: enp0s3
+kafka_broker_home: /usr/hdp/current/kafka-broker/
+
+# fastcapa settings
+fastcapa_portmask: 0x01
+fastcapa_kafka_config: /etc/fastcapa.conf
+fastcapa_topic: pcap
+fastcapa_burst_size: 32
+fastcapa_nb_rx_desc: 1024
+fastcapa_nb_rx_queue: 1
+fastcapa_tx_ring_size: 2048
+
+# dummy variables for pycapa's dependence on ambari_gather_facts
+cluster_name: dummy
+namenode_host: dummy
+core_site_tag: dummy
+hdfs_url: dummy
+kafka_broker_hosts: dummy
+kafka_broker_tag: dummy
+kafka_broker_port: dummy
+zookeeper_hosts: dummy
+zookeeper_tag: dummy
+zookeeper_port: dummy
+metron_hosts: dummy
+kibana_hosts: dummy
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile b/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
new file mode 100644
index 0000000..c62a0f1
--- /dev/null
+++ b/metron-deployment/development/fastcapa/centos-7.4/Vagrantfile
@@ -0,0 +1,72 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Vagrant.configure("2") do |config|
+
+ # enable hostmanager
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ #
+ # source
+ #
+ config.vm.define "source" do |node|
+
+ # host settings
+ node.vm.hostname = "source"
+ node.vm.box = "bento/centos-7.4"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.10", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 1024
+ vb.cpus = 1
+ end
+ end
+
+ #
+ # sink
+ #
+ config.vm.define "sink" do |node|
+
+ # host settings
+ node.vm.hostname = "sink"
+ node.vm.box = "bento/centos-7.4"
+ node.ssh.insert_key = "true"
+ node.vm.network :private_network, ip: "192.168.33.11", netmask: "255.255.255.0"
+
+ # provider
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = 4096
+ vb.cpus = 3
+
+ # network adapter settings; [Am79C970A|Am79C973|82540EM|82543GC|82545EM|virtio]
+ vb.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
+ vb.customize ["modifyvm", :id, "--nictype2","82545EM"]
+ end
+
+ # provision host
+ node.vm.provision :ansible do |ansible|
+ ansible.limit = "all"
+ ansible.playbook = "../playbook.yml"
+ ansible.extra_vars = "vars/main.yml"
+ end
+ end
+
+
+end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg b/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
new file mode 100644
index 0000000..dd78cfd
--- /dev/null
+++ b/metron-deployment/development/fastcapa/centos-7.4/ansible.cfg
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[defaults]
+host_key_checking = false
+library = ../../../ansible/extra_modules
+roles_path = ../../../ansible/roles
+pipelining = True
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml b/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml
new file mode 100644
index 0000000..563fc2a
--- /dev/null
+++ b/metron-deployment/development/fastcapa/centos-7.4/vars/main.yml
@@ -0,0 +1,52 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+# 0.9.4+ required for fastcapa
+librdkafka_version: 0.9.4
+librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
+
+dpdk_device: ["00:08.0"]
+dpdk_target: "x86_64-native-linuxapp-gcc"
+num_huge_pages: 512
+kafka_broker_url: source:9092
+zookeeper_url: source:2181
+pcap_replay_interface: enp0s3
+kafka_broker_home: /usr/hdp/current/kafka-broker/
+
+# fastcapa settings
+fastcapa_portmask: 0x01
+fastcapa_kafka_config: /etc/fastcapa.conf
+fastcapa_topic: pcap
+fastcapa_burst_size: 32
+fastcapa_nb_rx_desc: 1024
+fastcapa_nb_rx_queue: 1
+fastcapa_tx_ring_size: 2048
+
+# dummy variables for pycapa's dependence on ambari_gather_facts
+cluster_name: dummy
+namenode_host: dummy
+core_site_tag: dummy
+hdfs_url: dummy
+kafka_broker_hosts: dummy
+kafka_broker_tag: dummy
+kafka_broker_port: dummy
+zookeeper_hosts: dummy
+zookeeper_tag: dummy
+zookeeper_port: dummy
+metron_hosts: dummy
+kibana_hosts: dummy
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/playbook.yml b/metron-deployment/development/fastcapa/playbook.yml
new file mode 100644
index 0000000..e038691
--- /dev/null
+++ b/metron-deployment/development/fastcapa/playbook.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# the 'source' produces network traffic
+#
+- hosts: source
+ become: yes
+ roles:
+ - role: kafka-broker
+ - role: pcap_replay
+ - { role: pycapa, install_pycapa_service: False }
+ - { role: sensor-test-mode, pcap_replay: True, install_yaf: False, install_snort: False, install_pcap_replay: True }
+ tasks:
+ - service: name=pcap-replay state=started
+
+#
+# the 'sink' consumes network traffic
+#
+- hosts: sink
+ become: yes
+ roles:
+ - role: librdkafka
+ - role: fastcapa
+ tasks:
+ - service: name=fastcapa state=started
+
+#
+# validate the environment - needs to run on `source` node
+#
+- hosts: source
+ become: yes
+ tasks:
+ - include: tasks/validate-packets-sent.yml
+ - include: tasks/validate-messages-received.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml b/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml
new file mode 100644
index 0000000..5109a0e
--- /dev/null
+++ b/metron-deployment/development/fastcapa/tasks/validate-messages-received.yml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- set_fact:
+ kafka_consumer_cmd: "{{ kafka_broker_home}}/bin/kafka-console-consumer.sh --zookeeper {{ zookeeper_url }} --topic {{ fastcapa_topic }} --max-messages 10 --from-beginning"
+
+- name: Attempting to receive message(s) sent by fastcapa
+ shell: "{{ kafka_consumer_cmd }}"
+ async: 30
+ register: kafka_consumer
+
+- name: Waiting to receive message(s) sent by fastcapa
+ async_status: jid={{ kafka_consumer.ansible_job_id }}
+ register: job
+ until: job.finished
+ retries: 30
+
+- fail:
+ msg: "No messages received from fastcapa within timeout: {{ kafka_consumer_cmd }}"
+ when: job.finished != 1
+
+- debug: var=job
+
+- debug: msg="Successfully received a Kafka message from fastcapa!"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml b/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml
new file mode 100644
index 0000000..1b062d4
--- /dev/null
+++ b/metron-deployment/development/fastcapa/tasks/validate-packets-sent.yml
@@ -0,0 +1,41 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install tcpdump
+ yum: name=tcpdump
+
+- set_fact:
+ sniff_cmd: "tcpdump -i {{ pcap_replay_interface }} -c 10"
+
+- name: "Attempting to sniff packet(s)"
+ shell: "{{ sniff_cmd }}"
+ async: 30
+ register: sniffer
+
+- name: Waiting to receive packet(s)
+ async_status: jid={{ sniffer.ansible_job_id }}
+ register: job
+ until: job.finished
+ retries: 60
+
+- fail:
+ msg: "No packets received from pcap-replay within timeout: {{ sniff_cmd }}"
+ when: job.finished != 1
+
+- debug: var=job
+
+- debug: msg="Successfully received packets sent from pcap-replay!"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/README.md
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/README.md b/metron-deployment/development/ubuntu14/README.md
new file mode 100644
index 0000000..af7d3a3
--- /dev/null
+++ b/metron-deployment/development/ubuntu14/README.md
@@ -0,0 +1,106 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+Metron on Ubuntu 14
+===================
+
+This project fully automates the provisioning and deployment of Apache Metron and all necessary prerequisites on a single, virtualized host running Ubuntu 14.
+
+Metron is composed of many components and installing all of these on a single host, especially a virtualized one, will greatly stress the resources of the host. The host will require at least 8 GB of RAM and a fair amount of patience. It is highly recommended that you shut down all unnecessary services.
+
+Getting Started
+---------------
+
+### Prerequisites
+
+The computer used to deploy Apache Metron will need to have the following components installed.
+
+ - [Ansible](https://github.com/ansible/ansible) (2.0.0.2 or 2.2.2.0)
+ - [Docker](https://www.docker.com/community-edition)
+ - [Vagrant](https://www.vagrantup.com) 1.8+
+ - [Vagrant Hostmanager Plugin](https://github.com/devopsgroup-io/vagrant-hostmanager)
+ - [Virtualbox](https://virtualbox.org) 5.0+
+ - Python 2.7
+ - Maven 3.3.9
+ - C++11 compliant compiler, like [GCC](https://gcc.gnu.org/projects/cxx-status.html#cxx11)
+
+Running the following script can help validate whether you have all the prerequisites installed and running correctly.
+
+ ```
+ metron-deployment/scripts/platform-info.sh
+ ```
+
+#### How do I install these on MacOS?
+
+Any platform that supports these tools is suitable, but the following instructions cover installation on macOS. The easiest means of installing these tools on a Mac is to use the excellent [Homebrew](http://brew.sh/) project.
+
+1. Install Homebrew by following the instructions at [Homebrew](http://brew.sh/).
+
+1. Run the following command in a terminal to install all of the required tools.
+
+ ```
+ brew cask install vagrant virtualbox docker
+ brew cask install caskroom/versions/java8
+ brew install maven@3.3 git
+ pip install ansible==2.2.2.0
+ vagrant plugin install vagrant-hostmanager
+ open /Applications/Docker.app
+ ```
+
+### Deploy Metron
+
+1. Ensure that the Docker service is running.
+
+1. Deploy Metron
+
+ ```
+ cd metron-deployment/development/ubuntu14
+ vagrant up
+ ```
+
+ Should the process fail before completing the deployment, the following command will continue the deployment process without re-instantiating the host.
+
+ ```
+ vagrant provision
+ ```
+
+### Explore Metron
+
+Navigate to the following resources to explore your newly minted Apache Metron environment.
+
+* [Metron Alerts](http://node1:4201)
+* [Ambari](http://node1:8080)
+
+Connecting to the host through SSH is as simple as running the following command.
+
+```
+vagrant ssh
+```
+
+### Working with Metron
+
+In addition to re-running the entire provisioning play book, you may now re-run an individual Ansible tag or a collection of tags in the following ways. The following commands will re-run the `sensor-stubs` role on the Vagrant image. This will install and start the sensor stub components.
+
+```
+vagrant --ansible-tags="sensor-stubs" provision
+```
+
+Tags are listed in the playbooks, some frequently used tags:
++ `hdp-install` - Install HDP
++ `hdp-deploy` - Deploy and Start HDP Services (will start all Hadoop Services)
++ `sensors` - Deploy and start the sensors.
++ `sensor-stubs` - Deploy and start the sensor stubs.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/Vagrantfile
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/Vagrantfile b/metron-deployment/development/ubuntu14/Vagrantfile
new file mode 100644
index 0000000..55f0aea
--- /dev/null
+++ b/metron-deployment/development/ubuntu14/Vagrantfile
@@ -0,0 +1,87 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'getoptlong'
+
+ansibleTags=''
+ansibleSkipTags='sensors'
+
+begin
+ opts = GetoptLong.new(
+ [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ],
+ [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ]
+ )
+
+ opts.quiet = TRUE
+
+ opts.each do |opt, arg|
+ case opt
+ when '--ansible-tags'
+ ansibleTags=arg
+ when '--ansible-skip-tags'
+ ansibleSkipTags=arg
+ end
+ end
+rescue Exception => ignored
+#Ignore to allow other opts to be passed to Vagrant
+end
+
+puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != ''
+puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != ''
+
+hosts = [{
+ hostname: "node1",
+ ip: "192.168.66.121",
+ memory: "8192",
+ cpus: 4,
+ promisc: 2 # enables promisc on the 'Nth' network interface
+}]
+
+Vagrant.configure(2) do |config|
+
+ # host runs ubuntu
+ config.vm.box = "ubuntu/trusty64"
+ config.ssh.insert_key = true
+
+ # enable the hostmanager plugin
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ # host definition
+ hosts.each_with_index do |host, index|
+ config.vm.define host[:hostname] do |node|
+
+ # host settings
+ node.vm.hostname = host[:hostname]
+ node.vm.network "private_network", ip: host[:ip]
+
+ # vm settings
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = host[:memory]
+ vb.cpus = host[:cpus]
+ end
+ end
+ end
+
+ # provision the host with ansible
+ config.vm.provision :ansible do |ansible|
+ ansible.playbook = "ansible/playbook.yml"
+ ansible.sudo = true
+ ansible.tags = ansibleTags.split(",") if ansibleTags != ''
+ ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
+ ansible.inventory_path = "ansible/inventory"
+ end
+end
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible.cfg
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible.cfg b/metron-deployment/development/ubuntu14/ansible.cfg
new file mode 100644
index 0000000..ade50ae
--- /dev/null
+++ b/metron-deployment/development/ubuntu14/ansible.cfg
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[defaults]
+host_key_checking = false
+library = ../../ansible/extra_modules
+roles_path = ../../ansible/roles
+pipelining = True
+log_path = ./ansible.log
+
+
+# fix for "ssh throws 'unix domain socket too long' " problem
+[ssh_connection]
+control_path = %(directory)s/%%h-%%p-%%r
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
new file mode 100644
index 0000000..429d61a
--- /dev/null
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/group_vars/all
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# only need to build the "DEBS" for Ubuntu
+metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-debs"
+
+# ambari
+ambari_host: "{{ groups.ambari_master[0] }}"
+hdp_host_group: "{{ groups.ambari_slave }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
+cluster_type: single_node_vm
+ambari_server_mem: 512
+java_home: /usr/jdk64/jdk1.8.0_77
+
+# hbase
+pcap_hbase_table: pcap
+tracker_hbase_table: access_tracker
+threatintel_hbase_table: threatintel
+enrichment_hbase_table: enrichment
+
+# metron
+metron_version: 0.4.3
+metron_directory: /usr/metron/{{ metron_version }}
+bro_version: "2.5.2"
+fixbuf_version: "1.7.1"
+yaf_version: "2.8.0"
+daq_version: "2.0.6-1"
+pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+pycapa_home: "/opt/pycapa"
+snort_version: "2.9.8.0-1"
+snort_alert_csv_path: "/var/log/snort/alert.csv"
+threat_intel_bulk_load: False
+
+# data directories - only required to override defaults
+zookeeper_data_dir: "/data1/hadoop/zookeeper"
+namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
+namenode_name_dir: "/data1/hadoop/hdfs/namenode"
+datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
+journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
+nodemanager_local_dirs: "/data1/hadoop/yarn/local"
+timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
+timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
+nodemanager_log_dirs: "/data1/hadoop/yarn/log"
+jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
+storm_local_dir: "/data1/hadoop/storm"
+kafka_log_dirs: "/data1/kafka-log"
+elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
+
+# sensors
+sensor_test_mode: True
+install_pycapa: False
+install_bro: True
+install_snort: True
+install_yaf: False
+install_pcap_replay: True
+sniff_interface: eth1
+pcap_replay_interface: "{{ sniff_interface }}"
+pcapservice_port: 8081
+
+# search
+install_elasticsearch: True
+install_solr: False
+solr_collection_name: Metron
+solr_number_shards: 1
+solr_replication_factor: 1
+elasticsearch_transport_port: 9300
+## The elasticsearch_network_interface must be in the form specified for Elasticsearch, with leading and trailing underscores.
+elasticsearch_network_interface: _eth1_
+elasticsearch_web_port: 9200
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible/inventory/hosts
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/inventory/hosts b/metron-deployment/development/ubuntu14/ansible/inventory/hosts
new file mode 100644
index 0000000..9bd9ea1
--- /dev/null
+++ b/metron-deployment/development/ubuntu14/ansible/inventory/hosts
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[ambari_master]
+node1
+
+[ambari_slave]
+node1
+
+[metron]
+node1
+
+[search]
+node1
+
+[sensors]
+node1
+
+[pcap_server]
+node1
+
+[web]
+node1
+
+[zeppelin]
+node1
+
+[monit:children]
+sensors
+pcap_server
+
+[local]
+127.0.0.1
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/development/ubuntu14/ansible/playbook.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/development/ubuntu14/ansible/playbook.yml b/metron-deployment/development/ubuntu14/ansible/playbook.yml
new file mode 100644
index 0000000..fcbe99c
--- /dev/null
+++ b/metron-deployment/development/ubuntu14/ansible/playbook.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- hosts: all
+ pre_tasks:
+ - name: Update package cache on Ubuntu
+ apt: update_cache=yes
+ roles:
+ - role: enable-swap
+
+- include: ../../../ansible/playbooks/metron_full_install.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/extra_modules/ambari_cluster_state.py
----------------------------------------------------------------------
diff --git a/metron-deployment/extra_modules/ambari_cluster_state.py b/metron-deployment/extra_modules/ambari_cluster_state.py
deleted file mode 100644
index 0542526..0000000
--- a/metron-deployment/extra_modules/ambari_cluster_state.py
+++ /dev/null
@@ -1,395 +0,0 @@
-#!/usr/bin/python
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-DOCUMENTATION = '''
----
-module: ambari_cluster_state
-version_added: "2.1"
-author: Mark Bittmann (https://github.com/mbittmann)
-short_description: Create, delete, start or stop an ambari cluster
-description:
- - Create, delete, start or stop an ambari cluster
-options:
- host:
- description:
- The hostname for the ambari web server
- port:
- description:
- The port for the ambari web server
- username:
- description:
- The username for the ambari web server
- password:
- description:
- The name of the cluster in web server
- required: yes
- cluster_name:
- description:
- The name of the cluster in ambari
- required: yes
- cluster_state:
- description:
- The desired state for the ambari cluster ['present', 'absent', 'started', 'stopped']. Setting the cluster
- state to absent will first stop the cluster.
- required: yes
- blueprint_var:
- description:
- The path to the file defining the cluster blueprint and host mapping. Required when state == 'present'
- required: no
- blueprint_name:
- description:
- The name of the blueprint. Required when state == 'present'
- required: no
- wait_for_complete:
- description:
- Whether to wait for the request to complete before returning. Default is False.
- required: no
- requirements: [ 'requests']
-'''
-
-EXAMPLES = '''
-# must use full relative path to any files in stored in roles/role_name/files/
-- name: Create a new ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: present
- blueprint_var: roles/my_role/files/blueprint.yml
- blueprint_name: hadoop
- wait_for_complete: True
-- name: Start the ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: started
- wait_for_complete: True
-- name: Stop the ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: stopped
- wait_for_complete: True
-- name: Delete the ambari cluster
- ambari_cluster_state:
- host: localhost
- port: 8080
- username: admin
- password: admin
- cluster_name: my_cluster
- cluster_state: absent
-'''
-
-RETURN = '''
-results:
- description: The content of the requests object returned from the RESTful call
- returned: success
- type: string
-created_blueprint:
- description: Whether a blueprint was created
- returned: success
- type: boolean
-status:
- description: The status of the blueprint creation process
- returned: success
- type: string
-'''
-
-__author__ = 'mbittmann'
-
-import json
-try:
- import requests
-except ImportError:
- REQUESTS_FOUND = False
-else:
- REQUESTS_FOUND = True
-
-
-def main():
-
- argument_spec = dict(
- host=dict(type='str', default=None, required=True),
- port=dict(type='int', default=None, required=True),
- username=dict(type='str', default=None, required=True),
- password=dict(type='str', default=None, required=True),
- cluster_name=dict(type='str', default=None, required=True),
- cluster_state=dict(type='str', default=None, required=True,
- choices=['present', 'absent', 'started', 'stopped']),
- blueprint_var=dict(type='dict', required=False),
- blueprint_name=dict(type='str', default=None, required=False),
- configurations=dict(type='list', default=None, required=False),
- wait_for_complete=dict(default=False, required=False, type='bool'),
- )
-
- required_together = ['blueprint_var', 'blueprint_name']
-
- module = AnsibleModule(
- argument_spec=argument_spec,
- required_together=required_together
- )
-
- if not REQUESTS_FOUND:
- module.fail_json(
- msg='requests library is required for this module')
-
- p = module.params
-
- host = p.get('host')
- port = p.get('port')
- username = p.get('password')
- password = p.get('password')
- cluster_name = p.get('cluster_name')
- cluster_state = p.get('cluster_state')
- blueprint_name = p.get('blueprint_name')
- wait_for_complete = p.get('wait_for_complete')
-
- ambari_url = 'http://{0}:{1}'.format(host, port)
-
- try:
- if cluster_state in ['started', 'stopped']:
- if not cluster_exists(ambari_url, username, password, cluster_name):
- module.fail_json(msg="Cluster name {0} does not exist".format(cluster_name))
- state = ''
- if cluster_state == 'started':
- state = 'STARTED'
- elif cluster_state == 'stopped':
- state = 'INSTALLED'
-
- request = set_cluster_state(ambari_url, username, password, cluster_name, state)
- if wait_for_complete:
- try:
- request_id = json.loads(request.content)['Requests']['id']
- except ValueError:
- module.exit_json(changed=True, results=request.content)
- status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
- if status != 'COMPLETED':
- module.fail_json(msg="Request failed with status {0}".format(status))
- module.exit_json(changed=True, results=request.content)
- elif cluster_state == 'absent':
- if not cluster_exists(ambari_url, username, password, cluster_name):
- module.exit_json(changed=False, msg='Skipping. Cluster does not exist')
- if not can_delete_cluster(ambari_url, username, password, cluster_name):
- request = set_cluster_state(ambari_url, username, password, cluster_name, 'INSTALLED')
- request_id = json.loads(request.content)['Requests']['id']
- status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
- if status != 'COMPLETED':
- module.fail_json(msg="Request failed with status {0}".format(status))
- request = delete_cluster(ambari_url, username, password, cluster_name)
- module.exit_json(changed=True, results=request.content)
- elif cluster_state == 'present':
- if not p.get('blueprint_var') or not blueprint_name: # have neither name nor file
- module.fail_json(msg="Must provide blueprint_var and blueprint_name when cluster_state=='present'")
-
- blueprint_var = p.get('blueprint_var')
- blueprint, host_map = blueprint_var_to_ambari_converter(blueprint_var)
- created_blueprint = False
-
- if not blueprint_exists(ambari_url, username, password, blueprint_name):
- create_blueprint(ambari_url, username, password, blueprint_name, blueprint)
- created_blueprint = True
-
- if cluster_exists(ambari_url, username, password, cluster_name):
- module.exit_json(changed=False, msg='Cluster {0} already exists'.format(cluster_name),
- created_blueprint=created_blueprint)
-
- configurations = p.get('configurations')
- request = create_cluster(ambari_url, username, password, cluster_name, blueprint_name, configurations, host_map)
- request_id = json.loads(request.content)['Requests']['id']
- if wait_for_complete:
- status = wait_for_request_complete(ambari_url, username, password, cluster_name, request_id, 2)
- if status != 'COMPLETED':
- module.fail_json(msg="Request failed with status {0}".format(status))
- request_status = get_request_status(ambari_url, username, password, cluster_name, request_id)
- module.exit_json(changed=True, results=request.content,
- created_blueprint=created_blueprint, status=request_status)
-
- except requests.ConnectionError, e:
- module.fail_json(msg="Could not connect to Ambari client: " + str(e.message))
- except Exception, e:
- module.fail_json(msg="Ambari client exception occurred: " + str(e.message))
-
-
-def get_clusters(ambari_url, user, password):
- r = get(ambari_url, user, password, '/api/v1/clusters')
- if r.status_code != 200:
- msg = 'Could not get cluster list: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- clusters = json.loads(r.content)
- return clusters['items']
-
-
-def cluster_exists(ambari_url, user, password, cluster_name):
- clusters = get_clusters(ambari_url, user, password)
- return cluster_name in [item['Clusters']['cluster_name'] for item in clusters]
-
-
-def set_cluster_state(ambari_url, user, password, cluster_name, cluster_state):
- path = '/api/v1/clusters/{0}/services'.format(cluster_name)
- request = {"RequestInfo": {"context": "Setting cluster state"},
- "Body": {"ServiceInfo": {"state": "{0}".format(cluster_state)}}}
- payload = json.dumps(request)
- r = put(ambari_url, user, password, path, payload)
- if r.status_code not in [202, 200]:
- msg = 'Could not set cluster state: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def create_cluster(ambari_url, user, password, cluster_name, blueprint_name, configurations, hosts_json):
- path = '/api/v1/clusters/{0}'.format(cluster_name)
- data = json.dumps({'blueprint': blueprint_name, 'configurations': configurations, 'host_groups': hosts_json})
- f = open('cluster.log', 'w')
- f.write(data)
- f.close()
- r = post(ambari_url, user, password, path, data)
- if r.status_code != 202:
- msg = 'Could not create cluster: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def get_request_status(ambari_url, user, password, cluster_name, request_id):
- path = '/api/v1/clusters/{0}/requests/{1}'.format(cluster_name, request_id)
- r = get(ambari_url, user, password, path)
- if r.status_code != 200:
- msg = 'Could not get cluster request status: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- service = json.loads(r.content)
- return service['Requests']['request_status']
-
-
-def wait_for_request_complete(ambari_url, user, password, cluster_name, request_id, sleep_time):
- while True:
- status = get_request_status(ambari_url, user, password, cluster_name, request_id)
- if status == 'COMPLETED':
- return status
- elif status in ['FAILED', 'TIMEDOUT', 'ABORTED', 'SKIPPED_FAILED']:
- return status
- else:
- time.sleep(sleep_time)
-
-
-def can_delete_cluster(ambari_url, user, password, cluster_name):
- path = '/api/v1/clusters/{0}/services?ServiceInfo/state=STARTED'.format(cluster_name)
- r = get(ambari_url, user, password, path)
- items = json.loads(r.content)['items']
- return len(items) > 0
-
-
-def get_blueprints(ambari_url, user, password):
- path = '/api/v1/blueprints'
- r = get(ambari_url, user, password, path)
- if r.status_code != 200:
- msg = 'Could not get blueprint list: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
-
- services = json.loads(r.content)
- return services['items']
-
-
-def create_blueprint(ambari_url, user, password, blueprint_name, blueprint_data):
- data = json.dumps(blueprint_data)
- f = open('blueprint.log', 'w')
- f.write(data)
- f.close()
- path = "/api/v1/blueprints/" + blueprint_name
- r = post(ambari_url, user, password, path, data)
- if r.status_code != 201:
- msg = 'Could not create blueprint: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def blueprint_exists(ambari_url, user, password, blueprint_name):
- blueprints = get_blueprints(ambari_url, user, password)
- return blueprint_name in [item['Blueprints']['blueprint_name'] for item in blueprints]
-
-
-def delete_cluster(ambari_url, user, password, cluster_name):
- path = '/api/v1/clusters/{0}'.format(cluster_name)
- r = delete(ambari_url, user, password, path)
- if r.status_code != 200:
- msg = 'Could not delete cluster: request code {0}, \
- request message {1}'.format(r.status_code, r.content)
- raise Exception(msg)
- return r
-
-
-def get(ambari_url, user, password, path):
- r = requests.get(ambari_url + path, auth=(user, password))
- return r
-
-
-def put(ambari_url, user, password, path, data):
- headers = {'X-Requested-By': 'ambari'}
- r = requests.put(ambari_url + path, data=data, auth=(user, password), headers=headers)
- return r
-
-
-def post(ambari_url, user, password, path, data):
- headers = {'X-Requested-By': 'ambari'}
- r = requests.post(ambari_url + path, data=data, auth=(user, password), headers=headers)
- return r
-
-
-def delete(ambari_url, user, password, path):
- headers = {'X-Requested-By': 'ambari'}
- r = requests.delete(ambari_url + path, auth=(user, password), headers=headers)
- return r
-
-
-def blueprint_var_to_ambari_converter(blueprint_var):
- groups = blueprint_var['groups']
- new_groups = []
- host_map = []
- for group in groups:
- components = []
- for component in group['components']:
- components.append({'name': component})
- group['components'] = components
- hosts = group.pop('hosts')
- new_groups.append(group)
- this_host_map = dict()
- this_host_map['name'] = group['name']
- this_host_list = [{'fqdn': host} for host in hosts]
- this_host_map['hosts'] = this_host_list
- host_map.append(this_host_map)
- blueprint = dict()
- blueprint['configurations'] = blueprint_var['required_configurations']
- blueprint['host_groups'] = new_groups
- blueprint['Blueprints'] = {'stack_name': blueprint_var['stack_name'], 'stack_version': blueprint_var['stack_version']}
- return blueprint, host_map
-
-from ansible.module_utils.basic import *
-if __name__ == '__main__':
- main()
[16/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/bro/vars/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/bro/vars/main.yml b/metron-deployment/roles/bro/vars/main.yml
deleted file mode 100644
index 11d6c31..0000000
--- a/metron-deployment/roles/bro/vars/main.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-bro_home: /usr/local/bro
-bro_bin: "{{ bro_home }}/bin"
-bro_version: 2.5.2
-bro_daemon_log: /var/log/bro.log
-bro_topic: bro
-
-# Bro cronjob
-bro_crontab_minutes: 0-59/5
-bro_crontab_job: "{{ bro_home }}/bin/broctl cron"
-bro_clean_job: "rm -rf {{ bro_home }}/spool/tmp/*"
-
-# Bro kafka plugin
-metron_bro_plugin_kafka_version: "0.1"
-python27_bin: /opt/rh/python27/root/usr/bin
-python27_lib: /opt/rh/python27/root/usr/lib64
-git29_bin: /opt/rh/rh-git29/root/usr/bin
-httpd24_lib: /opt/rh/httpd24/root/usr/lib64
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/build-tools/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/build-tools/meta/main.yml b/metron-deployment/roles/build-tools/meta/main.yml
deleted file mode 100644
index ddf6aa9..0000000
--- a/metron-deployment/roles/build-tools/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/build-tools/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/build-tools/tasks/main.yml b/metron-deployment/roles/build-tools/tasks/main.yml
deleted file mode 100644
index c47ef43..0000000
--- a/metron-deployment/roles/build-tools/tasks/main.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install Build Tools
- yum: name={{ item }}
- with_items:
- - "@Development tools"
- - libdnet-devel
- - rpm-build
- - libpcap
- - libpcap-devel
- - pcre
- - pcre-devel
- - zlib
- - zlib-devel
- - glib2-devel
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/deployment-report/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/deployment-report/defaults/main.yml b/metron-deployment/roles/deployment-report/defaults/main.yml
deleted file mode 100644
index 0a8afb8..0000000
--- a/metron-deployment/roles/deployment-report/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-timeout_secs: 120
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/deployment-report/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/deployment-report/meta/main.yml b/metron-deployment/roles/deployment-report/meta/main.yml
deleted file mode 100644
index c3d807b..0000000
--- a/metron-deployment/roles/deployment-report/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/deployment-report/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/deployment-report/tasks/main.yml b/metron-deployment/roles/deployment-report/tasks/main.yml
deleted file mode 100644
index ac84ee8..0000000
--- a/metron-deployment/roles/deployment-report/tasks/main.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the 'License'); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an 'AS IS' BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Sanity check Metron web
- local_action: wait_for host="{{ groups.web[0] }}" port=5000 timeout="{{ timeout_secs }}"
-
-- name: Sanity check Ambari web
- local_action: wait_for host="{{ groups.ambari_master[0] }}" port="{{ ambari_port }}" timeout="{{ timeout_secs }}"
-
-- name: Known hosts groups
- debug: var=groups
-
-- set_fact:
- Success:
- - "Apache Metron deployed successfully"
- - " Metron @ http://{{ groups.web[0] }}:5000"
- - " Ambari @ http://{{ groups.ambari_master[0] }}:{{ ambari_port }}"
- - " Sensor Status @ http://{{ groups.sensors[0] }}:2812"
- - " Zookeeper @ {{ zookeeper_url }}"
- - " Kafka @ {{ kafka_broker_url }}"
- - For additional information, see https://metron.apache.org/'
-
-- debug: var=Success
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/epel/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/epel/tasks/main.yml b/metron-deployment/roles/epel/tasks/main.yml
deleted file mode 100644
index 0962e40..0000000
--- a/metron-deployment/roles/epel/tasks/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install EPEL repository
- yum: name=epel-release
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/defaults/main.yml b/metron-deployment/roles/fastcapa/defaults/main.yml
deleted file mode 100644
index d586046..0000000
--- a/metron-deployment/roles/fastcapa/defaults/main.yml
+++ /dev/null
@@ -1,42 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-# dpdk
-dpdk_home: "/usr/local/dpdk"
-dpdk_version: "17.08"
-dpdk_sdk: "/root/dpdk-{{ dpdk_version }}"
-dpdk_src_url: "http://fast.dpdk.org/rel/dpdk-{{ dpdk_version }}.tar.xz"
-dpdk_target: "x86_64-native-linuxapp-gcc"
-num_huge_pages: 512
-extra_cflags: -g
-
-# fastcapa
-fastcapa_work_dir: /root/fastcapa
-fastcapa_build_dir: "{{ fastcapa_work_dir }}/build/app/"
-fastcapa_prefix: /usr/local/bin
-fastcapa_ld_library_path: /usr/local/lib
-fastcapa_bin: fastcapa
-
-# fastcapa settings
-fastcapa_portmask: 0x01
-fastcapa_kafka_config: /etc/fastcapa.conf
-fastcapa_topic: pcap
-fastcapa_rx_burst_size: 32
-fastcapa_tx_burst_size: 256
-fastcapa_nb_rx_desc: 1024
-fastcapa_nb_rx_queue: 1
-fastcapa_tx_ring_size: 2048
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/meta/main.yml b/metron-deployment/roles/fastcapa/meta/main.yml
deleted file mode 100644
index d253e88..0000000
--- a/metron-deployment/roles/fastcapa/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - librdkafka
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/tasks/debug.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/debug.yml b/metron-deployment/roles/fastcapa/tasks/debug.yml
deleted file mode 100644
index 06f1526..0000000
--- a/metron-deployment/roles/fastcapa/tasks/debug.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Install debug utilities
- yum: name=yum-utils
- tags:
- - debug
-
- - name: Install debug symbols
- shell: debuginfo-install -y glibc glib2 zlib
- tags:
- - debug
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/dependencies.yml b/metron-deployment/roles/fastcapa/tasks/dependencies.yml
deleted file mode 100644
index 5955702..0000000
--- a/metron-deployment/roles/fastcapa/tasks/dependencies.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Install dependencies
- yum: name={{ item }}
- with_items:
- - "@Development tools"
- - pciutils
- - net-tools
- - glib2
- - glib2-devel
- - git
- - numactl-devel
-
- #
- # install prerequisite packages and the latest kernel headers. need to
- # ensure that the kernel headers match the current running kernel version.
- # if this is not the case, the DPDK build process will fail
- #
- - name: Install latest kernel headers and source
- yum: name={{ item }} state=latest
- with_items:
- - kernel
- - kernel-devel
- - kernel-headers
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/tasks/dpdk.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/dpdk.yml b/metron-deployment/roles/fastcapa/tasks/dpdk.yml
deleted file mode 100644
index a936570..0000000
--- a/metron-deployment/roles/fastcapa/tasks/dpdk.yml
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: "Download DPDK version {{ dpdk_version }}"
- unarchive:
- src: "{{ dpdk_src_url }}"
- dest: "/root"
- creates: "{{ dpdk_sdk }}"
- copy: no
-
- - name: "Configure DPDK for the target environment: {{ dpdk_target }}"
- shell: "make config T={{ dpdk_target }} DESTDIR={{ dpdk_home }}"
- args:
- chdir: "{{ dpdk_sdk }}"
- creates: "{{ dpdk_home }}"
-
- - name: "Turn on debug flags"
- lineinfile:
- dest: "{{ dpdk_sdk }}/config/common_linuxapp"
- regexp: 'DEBUG=n'
- line: 'DEBUG=y'
- tags:
- - debug
-
- - name: "Build DPDK for the target environment: {{ dpdk_target }}"
- shell: "make install T={{ dpdk_target }} DESTDIR={{ dpdk_home }} EXTRA_CFLAGS={{ extra_cflags }}"
- args:
- chdir: "{{ dpdk_sdk }}"
- creates: "{{ dpdk_home }}"
-
- - name: Load kernel modules to enable userspace IO
- shell: "{{ item }}"
- with_items:
- - modprobe uio_pci_generic
- - modprobe vfio-pci
-
- - name: Bind the device to the loaded kernel module(s)
- shell: "{{ dpdk_home }}/sbin/dpdk-devbind --force --bind=uio_pci_generic {{ item }}"
- with_items: "{{ dpdk_device }}"
-
- - name: Set useful environment variables
- lineinfile: "dest=/root/.bash_profile line={{ item }}"
- with_items:
- - "export RTE_SDK={{ dpdk_sdk }}"
- - "export RTE_TARGET={{ dpdk_target }}"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/fastcapa.yml b/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
deleted file mode 100644
index b555668..0000000
--- a/metron-deployment/roles/fastcapa/tasks/fastcapa.yml
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Distribute fastcapa
- copy: src=../../../metron-sensors/fastcapa dest={{ fastcapa_work_dir | dirname }} mode=0755
-
-- name: Build fastcapa
- shell: "{{ item }}"
- args:
- chdir: "{{ fastcapa_work_dir }}"
- with_items:
- - make
- environment:
- RTE_SDK: "{{ dpdk_sdk }}"
- RTE_TARGET: "{{ dpdk_target }}"
- LD_LIBRARY_PATH: "{{ fastcapa_ld_library_path }}"
-
-- name: Install fastcapa
- shell: "cp {{ fastcapa_build_dir }}/{{ fastcapa_bin }} {{ fastcapa_prefix }}"
- args:
- chdir: "{{ fastcapa_work_dir }}"
- creates: "{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
-
-- name: Deploy configuration
- template: src=fastcapa.conf dest={{ fastcapa_kafka_config }} mode=0755
-
-- name: Deploy service
- template: src=fastcapa dest=/etc/init.d/ mode=0755
-
-- name: Register the service with systemd
- shell: systemctl enable fastcapa
- when: ansible_distribution == "CentOS" and ansible_distribution_major_version == "7"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/tasks/kernel.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/kernel.yml b/metron-deployment/roles/fastcapa/tasks/kernel.yml
deleted file mode 100644
index cd4abe6..0000000
--- a/metron-deployment/roles/fastcapa/tasks/kernel.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-#
-# DPDK requires specific kernel boot parameters. set the params and reboot
-# the host, if the actual params differ from what is expected.
-#
----
- - set_fact:
- expected_kernel_params: "default_hugepagesz=1G hugepagesz=1G hugepages={{ num_huge_pages }} iommu=pt intel_iommu=on"
-
- - name: Check kernel boot parameters
- shell: "cat /proc/cmdline"
- register: actual_kernel_params
-
- - name: Alter kernel boot parameters
- lineinfile:
- dest: /etc/default/grub
- regexp: '^(GRUB_CMDLINE_LINUX=\"[^\"]+)\"$'
- line: '\1 {{ expected_kernel_params }}"'
- backrefs: yes
- when: not expected_kernel_params in actual_kernel_params.stdout
-
- - name: Update grub with kernel boot parameters
- shell: /sbin/grub2-mkconfig -o /boot/grub2/grub.cfg
- when: not expected_kernel_params in actual_kernel_params.stdout
-
- - name: Restart for modified kernel params
- command: shutdown -r now "modified kernel params"
- async: 0
- poll: 0
- ignore_errors: true
- when: not expected_kernel_params in actual_kernel_params.stdout
-
- - name: Wait for reboot of '{{ inventory_hostname }}'
- local_action: wait_for host={{ inventory_hostname }} state=started port=22 timeout=300 delay=10
- become: false
- when: not expected_kernel_params in actual_kernel_params.stdout
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/tasks/main.yml b/metron-deployment/roles/fastcapa/tasks/main.yml
deleted file mode 100644
index b98d557..0000000
--- a/metron-deployment/roles/fastcapa/tasks/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - include: dependencies.yml
- - include: kernel.yml
- - include: dpdk.yml
- - include: fastcapa.yml
- - include: debug.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/templates/fastcapa
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/templates/fastcapa b/metron-deployment/roles/fastcapa/templates/fastcapa
deleted file mode 100644
index f34c603..0000000
--- a/metron-deployment/roles/fastcapa/templates/fastcapa
+++ /dev/null
@@ -1,145 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# fastcapa daemon
-# chkconfig: 345 20 80
-# description: Packet capture probe
-# processname: fastcapa
-#
-
-export LD_LIBRARY_PATH="{{ fastcapa_ld_library_path }}"
-
-NAME="fastcapa"
-DESC="Metron network packet capture probe"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-LOGROOT=/var/log/fastcapa
-DAEMONLOG=$LOGROOT/$NAME-stdout.log
-DAEMONERR=$LOGROOT/$NAME-stderr.log
-NOW=`date`
-DAEMON_PATH="/root"
-
-PORT_MASK="{{ fastcapa_portmask }}"
-KAFKA_TOPIC="{{ fastcapa_topic }}"
-KAFKA_CONFIG="{{ fastcapa_kafka_config }}"
-RX_BURST_SIZE="{{ fastcapa_rx_burst_size }}"
-TX_BURST_SIZE="{{ fastcapa_tx_burst_size }}"
-NB_RX_DESC="{{ fastcapa_nb_rx_desc }}"
-NB_RX_QUEUE="{{ fastcapa_nb_rx_queue }}"
-TX_RING_SIZE="{{ fastcapa_tx_ring_size }}"
-
-DAEMON="{{ fastcapa_prefix }}/{{ fastcapa_bin }}"
-DAEMONOPTS+=" "
-DAEMONOPTS+=" -- "
-DAEMONOPTS+="-p $PORT_MASK "
-DAEMONOPTS+="-t $KAFKA_TOPIC "
-DAEMONOPTS+="-c $KAFKA_CONFIG "
-DAEMONOPTS+="-b $RX_BURST_SIZE "
-DAEMONOPTS+="-w $TX_BURST_SIZE "
-DAEMONOPTS+="-d $NB_RX_DESC "
-DAEMONOPTS+="-q $NB_RX_QUEUE "
-DAEMONOPTS+="-x $TX_RING_SIZE "
-
-case "$1" in
- start)
- printf "%-50s" "Starting $NAME..."
- echo "$NOW: Starting $NAME..." >> $DAEMONLOG
-
- mkdir -p $LOGROOT
- touch $DAEMONLOG
- touch $DAEMONERR
-
- cd $DAEMON_PATH
- echo "$DAEMON $DAEMONOPTS >> $DAEMONLOG 2> $DAEMONERR" >> $DAEMONLOG
-
- if [ -f $PIDFILE ]; then
- printf "%s\n" "Already running"
- else
- PID=`$DAEMON $DAEMONOPTS >> $DAEMONLOG 2> $DAEMONERR & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- fi
- ;;
-
- status)
- printf "%-50s" "Checking $NAME..."
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- echo "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- stop)
- printf "%-50s" "Stopping $NAME"
- PID=`cat $PIDFILE`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- while sleep 1
- echo -n "."
- kill -0 $PID >/dev/null 2>&1
- do
- kill -SIGINT $PID
- done
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- tail)
- tail -F $LOGROOT/*
- ;;
-
- kill)
- printf "%-50s" "Force killing $NAME"
- PID=`cat $PIDFILE`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- while sleep 1
- echo -n "."
- kill -0 $PID >/dev/null 2>&1
- do
- kill -SIGTERM $PID
- done
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart|kill|tail}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/fastcapa/templates/fastcapa.conf
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/fastcapa/templates/fastcapa.conf b/metron-deployment/roles/fastcapa/templates/fastcapa.conf
deleted file mode 100644
index 7d9eae4..0000000
--- a/metron-deployment/roles/fastcapa/templates/fastcapa.conf
+++ /dev/null
@@ -1,67 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#
-# kafka global settings
-#
-[kafka-global]
-
-# initial list of kafka brokers
-metadata.broker.list = {{ kafka_broker_url }}
-
-# identifies the client to kafka
-client.id = metron-fastcapa
-
-# max number of messages allowed on the producer queue
-queue.buffering.max.messages = 1000000
-
-# maximum time, in milliseconds, for buffering data on the producer queue
-queue.buffering.max.ms = 3000
-
-# compression codec = none, gzip or snappy
-compression.codec = snappy
-
-# maximum number of messages batched in one MessageSet (increase for better compression)
-batch.num.messages = 10
-
-# max times to retry sending a failed message set
-message.send.max.retries = 5
-
-# backoff time before retrying a message send
-retry.backoff.ms = 250
-
-# how often statistics are emitted; 0 = never
-statistics.interval.ms = 5000
-
-# only provide delivery reports for failed messages
-delivery.report.only.error = false
-
-#
-# kafka topic settings
-#
-[kafka-topic]
-
-# broker acks { 1 = leader ack, 0 = no acks, -1 = in sync replica ack }
-request.required.acks = 1
-
-# local message timeout. This value is only enforced locally and limits the time a
-# produced message waits for successful delivery. A time of 0 is infinite.
-message.timeout.ms = 10000
-
-# report offset of produced message back to application. The application must be
-# use the dr_msg_cb to retrieve the offset from rd_kafka_message_t.offset
-produce.offset.report = false
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/httplib2/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/httplib2/tasks/main.yml b/metron-deployment/roles/httplib2/tasks/main.yml
deleted file mode 100644
index 5502cf4..0000000
--- a/metron-deployment/roles/httplib2/tasks/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install python httplib2 dependency
- pip:
- name: httplib2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/java_jdk/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/java_jdk/defaults/main.yml b/metron-deployment/roles/java_jdk/defaults/main.yml
deleted file mode 100644
index 315bc0c..0000000
--- a/metron-deployment/roles/java_jdk/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-java_home: /usr/jdk64/jdk1.8.0_77
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/java_jdk/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/java_jdk/tasks/main.yml b/metron-deployment/roles/java_jdk/tasks/main.yml
deleted file mode 100644
index 999b9c1..0000000
--- a/metron-deployment/roles/java_jdk/tasks/main.yml
+++ /dev/null
@@ -1,34 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Check for java at "{{ java_home }}"
- stat: path="{{ java_home }}"
- register: jdk_dir
-
-- name: Alternatives link for java
- alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
- with_items:
- - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
- - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
- when: jdk_dir.stat.exists
-
-- name: Install openjdk
- yum: name={{item}}
- with_items:
- - java-1.8.0-openjdk
- - java-1.8.0-openjdk-devel
- when: not jdk_dir.stat.exists
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/kafka-broker/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/defaults/main.yml b/metron-deployment/roles/kafka-broker/defaults/main.yml
deleted file mode 100644
index b0f5a11..0000000
--- a/metron-deployment/roles/kafka-broker/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/kafka-broker/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/meta/main.yml b/metron-deployment/roles/kafka-broker/meta/main.yml
deleted file mode 100644
index 9587e79..0000000
--- a/metron-deployment/roles/kafka-broker/meta/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-dependencies:
- - libselinux-python
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/kafka-broker/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-broker/tasks/main.yml b/metron-deployment/roles/kafka-broker/tasks/main.yml
deleted file mode 100644
index db05cb0..0000000
--- a/metron-deployment/roles/kafka-broker/tasks/main.yml
+++ /dev/null
@@ -1,41 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Retrieve HDP repository definition
- get_url:
- url: "{{ hdp_repo_def }}"
- dest: /etc/yum.repos.d/hdp.repo
- mode: 0644
-
-- name: Install kafka
- yum: name={{item}}
- with_items:
- - java-1.8.0-openjdk
- - kafka
- - zookeeper-server
-
-- name: Create pid directories
- file: path={{ item }} state=directory mode=0755
- with_items:
- - /var/run/zookeeper
- - /var/run/kafka
-
-- name: Start zookeeper
- shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
-
-- name: Start kafka
- shell: /usr/hdp/current/kafka-broker/bin/kafka start
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/kafka-client/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-client/defaults/main.yml b/metron-deployment/roles/kafka-client/defaults/main.yml
deleted file mode 100644
index b0f5a11..0000000
--- a/metron-deployment/roles/kafka-client/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-hdp_repo_def: "http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.5.0.0/hdp.repo"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/kafka-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/kafka-client/tasks/main.yml b/metron-deployment/roles/kafka-client/tasks/main.yml
deleted file mode 100644
index 1674225..0000000
--- a/metron-deployment/roles/kafka-client/tasks/main.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Retrieve HDP repository definition
- get_url:
- url: "{{ hdp_repo_def }}"
- dest: /etc/yum.repos.d/hdp.repo
- mode: 0644
-
- - name: Install kafka
- yum:
- name: kafka
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/librdkafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/defaults/main.yml b/metron-deployment/roles/librdkafka/defaults/main.yml
deleted file mode 100644
index 063c22f..0000000
--- a/metron-deployment/roles/librdkafka/defaults/main.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-librdkafka_version: 0.9.4
-librdkafka_url: https://github.com/edenhill/librdkafka/archive/v0.9.4.tar.gz
-librdkafka_home: /usr/local
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/librdkafka/tasks/dependencies.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/tasks/dependencies.yml b/metron-deployment/roles/librdkafka/tasks/dependencies.yml
deleted file mode 100644
index 72ff907..0000000
--- a/metron-deployment/roles/librdkafka/tasks/dependencies.yml
+++ /dev/null
@@ -1,40 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install prerequisites
- yum: name={{ item }}
- with_items:
- - cmake
- - make
- - gcc
- - gcc-c++
- - flex
- - bison
- - libpcap
- - libpcap-devel
- - openssl-devel
- - python-devel
- - swig
- - zlib-devel
- - perl
- - cyrus-sasl
- - cyrus-sasl-devel
- - cyrus-sasl-gssapi
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/librdkafka/tasks/librdkafka.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/tasks/librdkafka.yml b/metron-deployment/roles/librdkafka/tasks/librdkafka.yml
deleted file mode 100644
index a7971d6..0000000
--- a/metron-deployment/roles/librdkafka/tasks/librdkafka.yml
+++ /dev/null
@@ -1,39 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Download librdkafka
- get_url:
- url: "{{ librdkafka_url }}"
- dest: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
-
-- name: Extract librdkafka tarball
- unarchive:
- src: "/tmp/librdkafka-{{ librdkafka_version }}.tar.gz"
- dest: /tmp
- copy: no
- creates: "/tmp/librdkafka-{{ librdkafka_version }}"
-
-- name: Compile and install librdkafka
- shell: "{{ item }}"
- args:
- chdir: "/tmp/librdkafka-{{ librdkafka_version }}"
- creates: "{{ librdkafka_home }}/lib/librdkafka.so"
- with_items:
- - rm -rf build/
- - "./configure --prefix={{ librdkafka_home }} --enable-sasl"
- - make
- - make install
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/librdkafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/librdkafka/tasks/main.yml b/metron-deployment/roles/librdkafka/tasks/main.yml
deleted file mode 100644
index 2144d7f..0000000
--- a/metron-deployment/roles/librdkafka/tasks/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: dependencies.yml
-- include: librdkafka.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/libselinux-python/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/libselinux-python/tasks/main.yml b/metron-deployment/roles/libselinux-python/tasks/main.yml
deleted file mode 100644
index 78f5a27..0000000
--- a/metron-deployment/roles/libselinux-python/tasks/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install libselinux-python
- yum:
- name: libselinux-python
- state: installed
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/load_web_templates/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/load_web_templates/meta/main.yml b/metron-deployment/roles/load_web_templates/meta/main.yml
deleted file mode 100644
index c3d807b..0000000
--- a/metron-deployment/roles/load_web_templates/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - ambari_gather_facts
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/load_web_templates/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/load_web_templates/tasks/main.yml b/metron-deployment/roles/load_web_templates/tasks/main.yml
deleted file mode 100644
index 3a91960..0000000
--- a/metron-deployment/roles/load_web_templates/tasks/main.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Load Kibana Dashboard
- command: >
- curl -s -w "%{http_code}" -u admin:admin -H "X-Requested-By: ambari" -X POST -d '{ "RequestInfo": { "context": "Install Kibana Dashboard from REST", "command": "LOAD_TEMPLATE"},"Requests/resource_filters": [{"service_name": "KIBANA","component_name": "KIBANA_MASTER","hosts" : "{{ kibana_hosts[0] }}"}]}' http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/requests
- args:
- warn: off
- register: result
- failed_when: "result.rc != 0 or '202' not in result.stdout"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-builder/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-builder/defaults/main.yml b/metron-deployment/roles/metron-builder/defaults/main.yml
deleted file mode 100644
index 07f22f4..0000000
--- a/metron-deployment/roles/metron-builder/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_build_dir: "{{ playbook_dir }}/../.."
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-builder/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-builder/tasks/main.yml b/metron-deployment/roles/metron-builder/tasks/main.yml
deleted file mode 100644
index d5fb517..0000000
--- a/metron-deployment/roles/metron-builder/tasks/main.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Build Metron
- local_action: shell cd {{ metron_build_dir }} && mvn clean package -DskipTests -T 2C -P HDP-2.5.0.0,mpack
- become: false
- run_once: true
-
-- name: Build Metron RPMs
- local_action: shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -P build-rpms
- become: false
- run_once: true
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-rpms/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-rpms/defaults/main.yml b/metron-deployment/roles/metron-rpms/defaults/main.yml
deleted file mode 100644
index 4a9d606..0000000
--- a/metron-deployment/roles/metron-rpms/defaults/main.yml
+++ /dev/null
@@ -1,18 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_rpm_glob: "{{ playbook_dir }}/../packaging/docker/rpm-docker/target/RPMS/noarch/*.rpm"
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-rpms/tasks/copy_rpms.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-rpms/tasks/copy_rpms.yml b/metron-deployment/roles/metron-rpms/tasks/copy_rpms.yml
deleted file mode 100644
index 3410b7f..0000000
--- a/metron-deployment/roles/metron-rpms/tasks/copy_rpms.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Copy Metron RPMs
- copy:
- src: "{{ item }}"
- dest: /localrepo
- owner: root
- mode: 0755
- with_fileglob:
- - "{{ metron_rpm_glob }}"
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-rpms/tasks/create_directory.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-rpms/tasks/create_directory.yml b/metron-deployment/roles/metron-rpms/tasks/create_directory.yml
deleted file mode 100644
index 0601625..0000000
--- a/metron-deployment/roles/metron-rpms/tasks/create_directory.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- name: Clean out localrepo
- file:
- state: absent
- path: /localrepo
-
-- name: Create localrepo directory
- file:
- path: /localrepo
- state: directory
- mode: 0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-rpms/tasks/create_repo.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-rpms/tasks/create_repo.yml b/metron-deployment/roles/metron-rpms/tasks/create_repo.yml
deleted file mode 100644
index e2ab00a..0000000
--- a/metron-deployment/roles/metron-rpms/tasks/create_repo.yml
+++ /dev/null
@@ -1,28 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Install createrepo
- package:
- name: createrepo
- state: installed
-
-- name: Create local repo with new packages
- shell: "createrepo /localrepo"
- args:
- creates: /localrepo/repodata
-
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron-rpms/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron-rpms/tasks/main.yml b/metron-deployment/roles/metron-rpms/tasks/main.yml
deleted file mode 100644
index 9ebb452..0000000
--- a/metron-deployment/roles/metron-rpms/tasks/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-
-- include: create_directory.yml
-- include: copy_rpms.yml
-- include: create_repo.yml
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template b/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template
deleted file mode 100644
index 3bb4633..0000000
--- a/metron-deployment/roles/metron_elasticsearch_templates/files/es_templates/error_index.template
+++ /dev/null
@@ -1,57 +0,0 @@
-{
- "template": "error_index*",
- "mappings": {
- "error_doc": {
- "_timestamp": {
- "enabled": true
- },
- "properties": {
- "exception": {
- "type": "string",
- "index": "not_analyzed"
- },
- "hostname": {
- "type": "string",
- "index": "not_analyzed"
- },
- "stack": {
- "type": "string",
- "index": "not_analyzed"
- },
- "timestamp": {
- "type": "date",
- "format": "epoch_millis"
- },
- "message": {
- "type": "string",
- "index": "not_analyzed"
- },
- "raw_message": {
- "type": "string",
- "index": "not_analyzed",
- "ignore_above": 8191
- },
- "raw_message_bytes": {
- "type": "binary",
- "index": "no"
- },
- "error_fields": {
- "type": "string",
- "index": "not_analyzed"
- },
- "error_hash": {
- "type": "string",
- "index": "not_analyzed"
- },
- "failed_sensor_type": {
- "type": "string",
- "index": "not_analyzed"
- },
- "error_type": {
- "type": "string",
- "index": "not_analyzed"
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/defaults/main.yml b/metron-deployment/roles/metron_pcapservice/defaults/main.yml
deleted file mode 100644
index c28938b..0000000
--- a/metron-deployment/roles/metron_pcapservice/defaults/main.yml
+++ /dev/null
@@ -1,29 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_version: 0.4.3
-metron_directory: /usr/metron/{{ metron_version }}
-pcapservice_jar_name: metron-api-{{ metron_version }}.jar
-pcapservice_jar_src: "{{ playbook_dir }}/../../metron-platform/metron-api/target/{{ pcapservice_jar_name }}"
-pcapservice_jar_dst: "{{ metron_directory }}/lib/{{ pcapservice_jar_name }}"
-pcapservice_port: 8081
-hbase_config_path: "/etc/hbase/conf"
-query_hdfs_path: "/tmp"
-pcap_hdfs_path: "/apps/metron/pcap"
-metron_pcapservice_logrotate_frequency: daily
-metron_pcapservice_logrotate_retention: 30
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/meta/main.yml b/metron-deployment/roles/metron_pcapservice/meta/main.yml
deleted file mode 100644
index ddf6aa9..0000000
--- a/metron-deployment/roles/metron_pcapservice/meta/main.yml
+++ /dev/null
@@ -1,19 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - java_jdk
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml b/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml
deleted file mode 100644
index b77c1ec..0000000
--- a/metron-deployment/roles/metron_pcapservice/tasks/config-hbase.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Fetch hbase-site.xml
- fetch: src=/etc/hbase/conf/hbase-site.xml dest=/tmp/hbase/conf/hbase-site.xml flat=yes
- delegate_to: "{{ groups.ambari_slave[0] }}"
-
- - name: Create hbase conf directory
- file: path=/etc/hbase/conf state=directory mode=0755
-
- - name: Copy hbase-site.xml
- copy: src=/tmp/hbase/conf/hbase-site.xml dest=/etc/hbase/conf/hbase-site.xml mode=0644
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/main.yml b/metron-deployment/roles/metron_pcapservice/tasks/main.yml
deleted file mode 100644
index ed33354..0000000
--- a/metron-deployment/roles/metron_pcapservice/tasks/main.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- include: pcapservice.yml
-
-- name: Create Logrotate Script for metron_pcapservice
- template:
- src: "metron-pcapservice-logrotate.yml"
- dest: "/etc/logrotate.d/metron-pcapservice"
- mode: 0644
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml b/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
deleted file mode 100644
index d0ac411..0000000
--- a/metron-deployment/roles/metron_pcapservice/tasks/pcapservice.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-- name: Create Metron streaming directories
- file: path={{ metron_directory }}/{{ item.name }} state=directory mode=0755
- with_items:
- - { name: 'lib'}
- - { name: 'config'}
-
-- name: Copy Metron pcapservice jar
- copy:
- src: "{{ pcapservice_jar_src }}"
- dest: "{{ pcapservice_jar_dst }}"
-
-- name: Install service script
- template: src=pcapservice dest=/etc/init.d/pcapservice mode=0755
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml b/metron-deployment/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
deleted file mode 100644
index 0293ade..0000000
--- a/metron-deployment/roles/metron_pcapservice/templates/metron-pcapservice-logrotate.yml
+++ /dev/null
@@ -1,27 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#Metron pcapService
-/var/log/metron_pcapservice.log {
- {{ metron_pcapservice_logrotate_frequency }}
- rotate {{ metron_pcapservice_logrotate_retention }}
- missingok
- notifempty
- copytruncate
- compress
-}
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_pcapservice/templates/pcapservice
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_pcapservice/templates/pcapservice b/metron-deployment/roles/metron_pcapservice/templates/pcapservice
deleted file mode 100644
index 054133d..0000000
--- a/metron-deployment/roles/metron_pcapservice/templates/pcapservice
+++ /dev/null
@@ -1,84 +0,0 @@
-#!/usr/bin/env bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# metron pcap service
-# chkconfig: 345 20 80
-# description: Metron PCAP Service Daemon
-# processname: pcapservice
-#
-NAME=pcapservice
-DESC="Metron pcap service"
-PIDFILE=/var/run/$NAME.pid
-SCRIPTNAME=/etc/init.d/$NAME
-LOGFILE="/var/log/metron_pcapservice.log"
-EXTRA_ARGS="${@:2}"
-DAEMON_PATH="/"
-DAEMON="/usr/bin/yarn jar"
-DAEMONOPTS="{{ pcapservice_jar_dst }} org.apache.metron.pcapservice.rest.PcapService -port {{ pcapservice_port }} -query_hdfs_path {{ query_hdfs_path }} -pcap_hdfs_path {{ pcap_hdfs_path }}"
-
-case "$1" in
- start)
- printf "%-50s" "Starting $NAME..."
-
- # kick-off the daemon
- cd $DAEMON_PATH
- PID=`$DAEMON $DAEMONOPTS >> $LOGFILE 2>&1 & echo $!`
- if [ -z $PID ]; then
- printf "%s\n" "Fail"
- else
- echo $PID > $PIDFILE
- printf "%s\n" "Ok"
- fi
- ;;
-
- status)
- printf "%-50s" "Checking $NAME..."
- if [ -f $PIDFILE ]; then
- PID=`cat $PIDFILE`
- if [ -z "`ps axf | grep ${PID} | grep -v grep`" ]; then
- printf "%s\n" "Process dead but pidfile exists"
- else
- echo "Running"
- fi
- else
- printf "%s\n" "Service not running"
- fi
- ;;
-
- stop)
- printf "%-50s" "Stopping $NAME"
- PID=`cat $PIDFILE`
- cd $DAEMON_PATH
- if [ -f $PIDFILE ]; then
- kill -HUP $PID
- printf "%s\n" "Ok"
- rm -f $PIDFILE
- else
- printf "%s\n" "pidfile not found"
- fi
- ;;
-
- restart)
- $0 stop
- $0 start
- ;;
-
- *)
- echo "Usage: $0 {status|start|stop|restart}"
- exit 1
-esac
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/defaults/main.yml b/metron-deployment/roles/metron_streaming/defaults/main.yml
deleted file mode 100644
index b9b34fe..0000000
--- a/metron-deployment/roles/metron_streaming/defaults/main.yml
+++ /dev/null
@@ -1,98 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_directory: /usr/metron/{{ metron_version }}
-metron_solr_jar_name: metron-solr-{{ metron_version }}.jar
-metron_elasticsearch_jar_name: metron-elasticsearch-{{ metron_version }}.jar
-metron_pcap_jar_name: metron-pcap-backend-{{ metron_version }}.jar
-metron_parsers_jar_name: metron-parsers-{{ metron_version }}.jar
-metron_enrichment_jar_name: metron-enrichment-{{ metron_version }}-uber.jar
-metron_profiler_jar_name: metron-profiler-{{ metron_version }}.jar
-
-# bundle names
-metron_common_bundle_name: metron-common-{{ metron_version }}-archive.tar.gz
-metron_pcap_bundle_name: metron-pcap-backend-{{ metron_version }}-archive.tar.gz
-metron_data_management_bundle_name: metron-data-management-{{ metron_version }}-archive.tar.gz
-metron_enrichment_bundle_name: metron-enrichment-{{ metron_version }}-archive.tar.gz
-metron_indexing_bundle_name: metron-indexing-{{ metron_version }}-archive.tar.gz
-metron_solr_bundle_name: metron-solr-{{ metron_version }}-archive.tar.gz
-metron_elasticsearch_bundle_name: metron-elasticsearch-{{ metron_version }}-archive.tar.gz
-metron_parsers_bundle_name: metron-parsers-{{ metron_version }}-archive.tar.gz
-metron_maas_bundle_name: metron-maas-service-{{ metron_version }}-archive.tar.gz
-metron_profiler_bundle_name: metron-profiler-{{ metron_version }}-archive.tar.gz
-metron_profiler_client_bundle_name: metron-profiler-client-{{ metron_version }}-archive.tar.gz
-
-# bundle paths
-metron_pcap_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-pcap-backend/target/{{ metron_pcap_bundle_name }}"
-metron_common_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-common/target/{{ metron_common_bundle_name }}"
-metron_data_management_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-data-management/target/{{ metron_data_management_bundle_name }}"
-metron_enrichment_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-enrichment/target/{{ metron_enrichment_bundle_name }}"
-metron_indexing_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-indexing/target/{{ metron_indexing_bundle_name }}"
-metron_solr_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-solr/target/{{ metron_solr_bundle_name }}"
-metron_elasticsearch_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-elasticsearch/target/{{ metron_elasticsearch_bundle_name }}"
-metron_parsers_bundle_path: "{{ playbook_dir }}/../../metron-platform/metron-parsers/target/{{ metron_parsers_bundle_name }}"
-metron_maas_bundle_path: "{{ playbook_dir }}/../../metron-analytics/metron-maas-service/target/{{ metron_maas_bundle_name }}"
-metron_profiler_bundle_path: "{{ playbook_dir }}/../../metron-analytics/metron-profiler/target/{{ metron_profiler_bundle_name }}"
-metron_profiler_client_bundle_path: "{{ playbook_dir }}/../../metron-analytics/metron-profiler-client/target/{{ metron_profiler_client_bundle_name }}"
-
-
-# configuration paths
-config_path: "{{ metron_directory }}/config"
-zookeeper_config_path: "{{ config_path }}/zookeeper"
-zookeeper_global_config_path: "{{ zookeeper_config_path }}/global.json"
-metron_pcap_properties_config_path: "{{ metron_directory }}/config/pcap.properties"
-metron_solr_properties_config_path: "{{ metron_directory }}/config/solr.properties"
-metron_elasticsearch_properties_config_path: "{{ metron_directory }}/config/elasticsearch.properties"
-metron_enrichment_properties_config_path: "{{ metron_directory }}/config/enrichment.properties"
-metron_parsers_properties_config_path: "{{ metron_directory }}/config/parsers.properties"
-metron_profiler_properties_config_path: "{{ metron_directory }}/config/profiler.properties"
-
-hbase_config_path: "/etc/hbase/conf"
-hdfs_config_path: "/etc/hadoop/conf"
-pcap_hdfs_path: "/apps/metron/pcap"
-geo_hdfs_path: "/apps/metron/geo/default"
-
-threat_intel_bulk_load: True
-threat_intel_bin: "{{ metron_directory }}/bin/flatfile_loader.sh"
-threat_intel_work_dir: /tmp/ti_bulk
-threat_intel_csv_filename: "threat_ip.csv"
-threat_intel_csv_filepath: "{{ threat_intel_csv_filename }}"
-
-pycapa_topic: pcap
-bro_topic: bro
-yaf_topic: yaf
-snort_topic: snort
-enrichments_topic: enrichments
-
-hdfs_retention_days: 30
-hdfs_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/indexing/indexed/bro_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-hdfs-bro-purge.log 2>&1"
-hdfs_yaf_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/indexing/indexed/yaf_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-hdfs-yaf-purge.log 2>&1"
-hdfs_snort_purge_cronjob: "{{ metron_directory }}/bin/prune_hdfs_files.sh -f {{ hdfs_url }} -g '/apps/metron/indexing/indexed/snort_doc/*enrichment-*' -s $(date -d '{{ hdfs_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-hdfs-snort-purge.log 2>&1"
-
-elasticsearch_config_path: /etc/elasticsearch
-elasticsearch_cluster_name: metron
-elasticsearch_transport_port: 9300
-
-es_retention_days: 30
-es_bro_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p bro_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/bro-purge/cron-es-bro-purge.log 2>&1"
-es_yaf_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p yaf_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/yaf-purge/cron-es-yaf-purge.log 2>&1"
-es_snort_purge_cronjob: "{{ metron_directory }}/bin/prune_elasticsearch_indices.sh -z {{ zookeeper_url }} -p yaf_index_ -s $(date -d '{{ es_retention_days }} days ago' +%m/%d/%Y) -n 1 >> /var/log/snort-purge/cron-es-snort-purge.log 2>&1"
-
-metron_hdfs_output_dir: "/apps/metron"
-metron_hdfs_rotation_policy: org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy
-metron_hdfs_rotation_policy_count: 1
-metron_hdfs_rotation_policy_units: DAYS
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/files/extractor.json
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/files/extractor.json b/metron-deployment/roles/metron_streaming/files/extractor.json
deleted file mode 100644
index 545202a..0000000
--- a/metron-deployment/roles/metron_streaming/files/extractor.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "config": {
- "columns": {
- "ip": 0
- },
- "indicator_column": "ip",
- "type" : "malicious_ip",
- "separator": ","
- },
- "extractor": "CSV"
-}
-
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/metron_streaming/meta/main.yml
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/metron_streaming/meta/main.yml b/metron-deployment/roles/metron_streaming/meta/main.yml
deleted file mode 100644
index de2f386..0000000
--- a/metron-deployment/roles/metron_streaming/meta/main.yml
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-dependencies:
- - ambari_gather_facts
- - java_jdk
- - libselinux-python
[35/50] [abbrv] metron git commit: METRON-1439 Turn off git pager in
platform-info script (justinleet) closes apache/metron#919
Posted by rm...@apache.org.
METRON-1439 Turn off git pager in platform-info script (justinleet) closes apache/metron#919
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/567d106b
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/567d106b
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/567d106b
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 567d106b1fbe86e828f3c4068024df42b2f4ddc0
Parents: 124becd
Author: justinleet <ju...@gmail.com>
Authored: Thu Feb 1 13:46:55 2018 -0500
Committer: leet <le...@apache.org>
Committed: Thu Feb 1 13:46:55 2018 -0500
----------------------------------------------------------------------
metron-deployment/scripts/platform-info.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/567d106b/metron-deployment/scripts/platform-info.sh
----------------------------------------------------------------------
diff --git a/metron-deployment/scripts/platform-info.sh b/metron-deployment/scripts/platform-info.sh
index 98e0bcf..fd2248a 100755
--- a/metron-deployment/scripts/platform-info.sh
+++ b/metron-deployment/scripts/platform-info.sh
@@ -36,11 +36,11 @@ if [ "$IS_GIT_REPO" == "true" ]; then
# last commit
echo "--"
- git log -n 1
+ git --no-pager log -n 1
# local changes since last commit
echo "--"
- git diff --stat
+ git --no-pager diff --stat
fi
# ansible
[07/50] [abbrv] metron git commit: METRON-1398 Exclude the
basic-error-controller from being added to the swagger description (MohanDV
via merrimanr) closes apache/metron#898
Posted by rm...@apache.org.
METRON-1398 Exclude the basic-error-controller from being added to the swagger description (MohanDV via merrimanr) closes apache/metron#898
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/92c2e22e
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/92c2e22e
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/92c2e22e
Branch: refs/heads/feature/METRON-1344-test-infrastructure
Commit: 92c2e22e6cd50c122c9da1e8f6190a812eb01efb
Parents: a421df2
Author: MohanDV <mo...@gmail.com>
Authored: Fri Jan 26 08:23:24 2018 -0600
Committer: merrimanr <me...@apache.org>
Committed: Fri Jan 26 08:23:24 2018 -0600
----------------------------------------------------------------------
.../main/java/org/apache/metron/rest/config/SwaggerConfig.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metron/blob/92c2e22e/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
----------------------------------------------------------------------
diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
index 9e82196..564ff32 100644
--- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
+++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/SwaggerConfig.java
@@ -19,6 +19,7 @@ package org.apache.metron.rest.config;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
+import org.springframework.web.bind.annotation.RestController;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.spi.DocumentationType;
@@ -32,7 +33,7 @@ public class SwaggerConfig {
public Docket api() {
return new Docket(DocumentationType.SWAGGER_2)
.select()
- .apis(RequestHandlerSelectors.any())
+ .apis(RequestHandlerSelectors.withClassAnnotation(RestController.class))
.paths(PathSelectors.any())
.build();
}
[12/50] [abbrv] metron git commit: METRON-1370 Create Full Dev
Equivalent for Ubuntu (nickwallen via cestella) closes
apache/incubator-metron#903
Posted by rm...@apache.org.
http://git-wip-us.apache.org/repos/asf/metron/blob/6f267991/metron-deployment/roles/sensor-stubs/files/snort.out
----------------------------------------------------------------------
diff --git a/metron-deployment/roles/sensor-stubs/files/snort.out b/metron-deployment/roles/sensor-stubs/files/snort.out
deleted file mode 100644
index 32f9a53..0000000
--- a/metron-deployment/roles/sensor-stubs/files/snort.out
+++ /dev/null
@@ -1,27404 +0,0 @@
-01/11/17-20:49:18.107168 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E396BFC,0x56900BB6,,0x1000,64,10,23403,76,77824,,,,
-01/11/17-20:49:18.107195 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900BB6,0x1E396C14,,0x1F13,64,16,3894,52,53248,,,,
-01/11/17-20:49:18.107396 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396C14,0x56900BCE,,0xFFF,64,8,40177,52,53248,,,,
-01/11/17-20:49:18.107510 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E396C14,0x56900BCE,,0x1000,64,10,53179,204,208896,,,,
-01/11/17-20:49:18.109884 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CAC,0x56900C16,,0xFFD,64,8,30920,52,53248,,,,
-01/11/17-20:49:18.132533 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CAC,0x56900C7E,,0xFFC,64,8,44619,52,53248,,,,
-01/11/17-20:49:18.132540 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CAC,0x56900CAE,,0xFFE,64,8,55021,52,53248,,,,
-01/11/17-20:49:18.132751 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E396CAC,0x56900CAE,,0x1000,64,10,24370,76,77824,,,,
-01/11/17-20:49:18.154679 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396CC4,0x56900CAE,,0x1000,64,10,50877,92,94208,,,,
-01/11/17-20:49:18.154802 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900CAE,0x1E396CEC,,0x1F13,64,16,3899,52,53248,,,,
-01/11/17-20:49:18.155051 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396CEC,0x56900CC6,,0xFFF,64,8,6931,52,53248,,,,
-01/11/17-20:49:18.155074 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E396CEC,0x56900CC6,,0x1000,64,10,50478,116,118784,,,,
-01/11/17-20:49:18.165247 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396D2C,0x56900CF6,,0xFFE,64,8,57108,52,53248,,,,
-01/11/17-20:49:18.315958 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D2C,0x56900CF6,,0x1000,64,10,31240,92,94208,,,,
-01/11/17-20:49:18.315973 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D2C,0x56900CF6,,0x1000,64,8,51328,92,94208,,,,
-01/11/17-20:49:18.315979 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4E,***A****,0x56900CF6,0x1E396D54,,0x1F13,64,16,3902,64,65536,,,,
-01/11/17-20:49:18.317192 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396D54,0x56900D0E,,0xFFF,64,8,8422,52,53248,,,,
-01/11/17-20:49:18.317376 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D54,0x56900D0E,,0x1000,64,10,26873,92,94208,,,,
-01/11/17-20:49:18.317729 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396D7C,0x56900D26,,0xFFF,64,8,63956,52,53248,,,,
-01/11/17-20:49:18.318013 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396D7C,0x56900D26,,0x1000,64,10,21452,92,94208,,,,
-01/11/17-20:49:18.318328 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396DA4,0x56900D3E,,0xFFF,64,8,64316,52,53248,,,,
-01/11/17-20:49:18.323894 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E396DA4,0x56900D3E,,0x1000,64,10,49851,92,94208,,,,
-01/11/17-20:49:18.324357 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E396DCC,0x56900D56,,0xFFF,64,8,54379,52,53248,,,,
-01/11/17-20:49:18.325050 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E396DCC,0x56900D56,,0x1000,64,10,63932,1500,225300,,,,
-01/11/17-20:49:18.325060 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E397374,0x56900D56,,0x1000,64,10,34229,1060,36880,,,,
-01/11/17-20:49:18.325065 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E397764,,0x1F13,64,16,3907,52,53248,,,,
-01/11/17-20:49:18.325419 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E397764,0x56900D56,,0x1000,64,10,59177,1500,225300,,,,
-01/11/17-20:49:18.325425 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E397D0C,0x56900D56,,0x1000,64,10,49253,1500,225300,,,,
-01/11/17-20:49:18.325427 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E3982B4,,0x1F13,64,16,3908,52,53248,,,,
-01/11/17-20:49:18.325439 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4A,***AP***,0x1E3982B4,0x56900D56,,0x1000,64,10,12980,60,61440,,,,
-01/11/17-20:49:18.325798 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3982BC,0x56900D56,,0x1000,64,10,17183,1500,225300,,,,
-01/11/17-20:49:18.325803 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E398864,,0x1F13,64,16,3909,52,53248,,,,
-01/11/17-20:49:18.325813 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E398864,0x56900D56,,0x1000,64,10,10585,644,135176,,,,
-01/11/17-20:49:18.326167 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E398AB4,0x56900D56,,0x1000,64,10,64378,1500,225300,,,,
-01/11/17-20:49:18.326173 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E39905C,,0x1F13,64,16,3910,52,53248,,,,
-01/11/17-20:49:18.326183 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E39905C,0x56900D56,,0x1000,64,10,36369,900,135180,,,,
-01/11/17-20:49:18.326185 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3993AC,0x56900D56,,0x1000,64,10,42540,92,94208,,,,
-01/11/17-20:49:18.330409 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D56,0x1E3993D4,,0x1F13,64,16,3911,52,53248,,,,
-01/11/17-20:49:18.331204 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3993D4,0x56900D6E,,0xFFF,64,8,42136,52,53248,,,,
-01/11/17-20:49:18.331625 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3993D4,0x56900D6E,,0x1000,64,10,48207,1500,225300,,,,
-01/11/17-20:49:18.331641 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E39997C,0x56900D6E,,0x1000,64,10,21064,292,36868,,,,
-01/11/17-20:49:18.331645 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E399A6C,,0x1F13,64,16,3913,52,53248,,,,
-01/11/17-20:49:18.331923 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E399A6C,0x56900D6E,,0x1000,64,10,15105,1500,225300,,,,
-01/11/17-20:49:18.331938 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E39A014,0x56900D6E,,0x1000,64,10,1341,468,217092,,,,
-01/11/17-20:49:18.331941 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39A1B4,,0x1F13,64,16,3914,52,53248,,,,
-01/11/17-20:49:18.332270 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39A1B4,0x56900D6E,,0x1000,64,10,10751,1500,225300,,,,
-01/11/17-20:49:18.332284 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E39A75C,0x56900D6E,,0x1000,64,10,2115,164,167936,,,,
-01/11/17-20:49:18.332286 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39A7CC,,0x1F13,64,16,3915,52,53248,,,,
-01/11/17-20:49:18.332650 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39A7CC,0x56900D6E,,0x1000,64,10,10561,1500,225300,,,,
-01/11/17-20:49:18.332655 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E39AD74,0x56900D6E,,0x1000,64,10,42742,788,20492,,,,
-01/11/17-20:49:18.332658 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39B054,,0x1F13,64,16,3916,52,53248,,,,
-01/11/17-20:49:18.333034 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39B054,0x56900D6E,,0x1000,64,10,56541,1500,225300,,,,
-01/11/17-20:49:18.333048 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E39B5FC,0x56900D6E,,0x1000,64,10,25591,948,184332,,,,
-01/11/17-20:49:18.333052 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D6E,0x1E39B97C,,0x1F13,64,16,3917,52,53248,,,,
-01/11/17-20:49:18.333403 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39B97C,0x56900D86,,0xFFF,64,8,24503,52,53248,,,,
-01/11/17-20:49:18.333409 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39B97C,0x56900D86,,0x1000,64,10,63480,1500,225300,,,,
-01/11/17-20:49:18.333421 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E39BF24,0x56900D86,,0x1000,64,10,61701,964,200716,,,,
-01/11/17-20:49:18.333424 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D86,0x1E39C2B4,,0x1F13,64,16,3919,52,53248,,,,
-01/11/17-20:49:18.333816 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39C2B4,0x56900D86,,0x1000,64,10,25421,1500,225300,,,,
-01/11/17-20:49:18.333820 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E39C85C,0x56900D86,,0x1000,64,10,25178,756,249864,,,,
-01/11/17-20:49:18.333822 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900D86,0x1E39CB1C,,0x1F13,64,16,3920,52,53248,,,,
-01/11/17-20:49:18.334060 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB1C,0x56900D9E,,0xFFF,64,8,59095,52,53248,,,,
-01/11/17-20:49:18.334219 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CB1C,0x56900D9E,,0x1000,64,10,61490,92,94208,,,,
-01/11/17-20:49:18.334426 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB44,0x56900DB6,,0xFFF,64,8,62611,52,53248,,,,
-01/11/17-20:49:18.334608 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CB44,0x56900DB6,,0x1000,64,10,33533,76,77824,,,,
-01/11/17-20:49:18.335128 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB5C,0x56900DFE,,0xFFD,64,8,44344,52,53248,,,,
-01/11/17-20:49:18.335229 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CB5C,0x56900DFE,,0x1000,64,10,65063,76,77824,,,,
-01/11/17-20:49:18.348745 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CB74,0x56900DFE,,0x1000,64,10,45295,92,94208,,,,
-01/11/17-20:49:18.348795 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900DFE,0x1E39CB9C,,0x1F13,64,16,3924,52,53248,,,,
-01/11/17-20:49:18.349013 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CB9C,0x56900E16,,0xFFF,64,8,20224,52,53248,,,,
-01/11/17-20:49:18.349132 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E39CB9C,0x56900E16,,0x1000,64,10,37526,556,45064,,,,
-01/11/17-20:49:18.352001 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900E5E,,0xFFD,64,8,37910,52,53248,,,,
-01/11/17-20:49:18.383171 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900E96,,0xFFE,64,8,36703,52,53248,,,,
-01/11/17-20:49:18.441681 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900ECE,,0xFFE,64,8,55590,52,53248,,,,
-01/11/17-20:49:18.451874 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CD94,0x56900F2E,,0xFFD,64,8,19749,52,53248,,,,
-01/11/17-20:49:18.452037 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CD94,0x56900F2E,,0x1000,64,10,168,76,77824,,,,
-01/11/17-20:49:18.491519 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900F2E,0x1E39CDAC,,0x1F13,64,16,3930,52,53248,,,,
-01/11/17-20:49:28.469092 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CDAC,0x56900F2E,,0x1000,64,10,30215,76,77824,,,,
-01/11/17-20:49:28.469122 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56900F2E,0x1E39CDC4,,0x1F13,64,16,3931,52,53248,,,,
-01/11/17-20:49:28.469442 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CDC4,0x56900F46,,0xFFF,64,8,17853,52,53248,,,,
-01/11/17-20:49:28.469543 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E39CDC4,0x56900F46,,0x1000,64,10,25675,204,208896,,,,
-01/11/17-20:49:28.470972 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE5C,0x56900F8E,,0xFFD,64,8,39640,52,53248,,,,
-01/11/17-20:49:28.482209 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE5C,0x56900FE6,,0xFFD,64,8,45105,52,53248,,,,
-01/11/17-20:49:28.482287 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE5C,0x56901016,,0xFFE,64,8,44659,52,53248,,,,
-01/11/17-20:49:28.482352 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E39CE5C,0x56901016,,0x1000,64,10,64668,76,77824,,,,
-01/11/17-20:49:28.500940 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CE74,0x56901016,,0x1000,64,10,18360,92,94208,,,,
-01/11/17-20:49:28.501082 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901016,0x1E39CE9C,,0x1F13,64,16,3936,52,53248,,,,
-01/11/17-20:49:28.501262 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CE9C,0x5690102E,,0xFFF,64,8,33893,52,53248,,,,
-01/11/17-20:49:28.501342 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E39CE9C,0x5690102E,,0x1000,64,10,50754,116,118784,,,,
-01/11/17-20:49:28.502358 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CEDC,0x5690105E,,0xFFE,64,8,42796,52,53248,,,,
-01/11/17-20:49:28.502487 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CEDC,0x5690105E,,0x1000,64,10,9919,92,94208,,,,
-01/11/17-20:49:28.512827 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF04,0x56901076,,0xFFF,64,8,49447,52,53248,,,,
-01/11/17-20:49:28.512959 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CF04,0x56901076,,0x1000,64,10,29403,92,94208,,,,
-01/11/17-20:49:28.513418 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF2C,0x5690108E,,0xFFF,64,8,19625,52,53248,,,,
-01/11/17-20:49:28.513717 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CF2C,0x5690108E,,0x1000,64,10,21710,92,94208,,,,
-01/11/17-20:49:28.514000 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF54,0x569010A6,,0xFFF,64,8,10409,52,53248,,,,
-01/11/17-20:49:28.519867 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39CF54,0x569010A6,,0x1000,64,10,7830,92,94208,,,,
-01/11/17-20:49:28.520589 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39CF7C,0x569010BE,,0xFFF,64,8,6502,52,53248,,,,
-01/11/17-20:49:28.520731 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39CF7C,0x569010BE,,0x1000,64,10,51267,1500,225300,,,,
-01/11/17-20:49:28.520747 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E39D524,0x569010BE,,0x1000,64,10,15295,1060,36880,,,,
-01/11/17-20:49:28.520750 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39D914,,0x1F13,64,16,3943,52,53248,,,,
-01/11/17-20:49:28.521160 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39D914,0x569010BE,,0x1000,64,10,42479,1500,225300,,,,
-01/11/17-20:49:28.521166 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39DEBC,0x569010BE,,0x1000,64,10,65233,1500,225300,,,,
-01/11/17-20:49:28.521168 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39E464,,0x1F13,64,16,3944,52,53248,,,,
-01/11/17-20:49:28.521179 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4A,***AP***,0x1E39E464,0x569010BE,,0x1000,64,10,6499,60,61440,,,,
-01/11/17-20:49:28.521543 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39E46C,0x569010BE,,0x1000,64,10,36882,1500,225300,,,,
-01/11/17-20:49:28.521548 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39EA14,,0x1F13,64,16,3945,52,53248,,,,
-01/11/17-20:49:28.521559 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E39EA14,0x569010BE,,0x1000,64,10,49793,644,135176,,,,
-01/11/17-20:49:28.522079 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39EC64,0x569010BE,,0x1000,64,10,56708,1500,225300,,,,
-01/11/17-20:49:28.522084 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39F20C,,0x1F13,64,16,3946,52,53248,,,,
-01/11/17-20:49:28.522105 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E39F20C,0x569010BE,,0x1000,64,10,33698,900,135180,,,,
-01/11/17-20:49:28.522107 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E39F55C,0x569010BE,,0x1000,64,10,15201,92,94208,,,,
-01/11/17-20:49:28.522148 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010BE,0x1E39F584,,0x1F13,64,16,3947,52,53248,,,,
-01/11/17-20:49:28.522763 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E39F584,0x569010D6,,0xFFF,64,8,64832,52,53248,,,,
-01/11/17-20:49:28.522911 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39F584,0x569010D6,,0x1000,64,10,18426,1500,225300,,,,
-01/11/17-20:49:28.522933 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E39FB2C,0x569010D6,,0x1000,64,10,55360,292,36868,,,,
-01/11/17-20:49:28.522938 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E39FC1C,,0x1F13,64,16,3949,52,53248,,,,
-01/11/17-20:49:28.523205 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E39FC1C,0x569010D6,,0x1000,64,10,4284,1500,225300,,,,
-01/11/17-20:49:28.523209 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E3A01C4,0x569010D6,,0x1000,64,10,12946,468,217092,,,,
-01/11/17-20:49:28.523211 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A0364,,0x1F13,64,16,3950,52,53248,,,,
-01/11/17-20:49:28.523456 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A0364,0x569010D6,,0x1000,64,10,7500,1500,225300,,,,
-01/11/17-20:49:28.523459 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E3A090C,0x569010D6,,0x1000,64,10,40144,164,167936,,,,
-01/11/17-20:49:28.523461 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A097C,,0x1F13,64,16,3951,52,53248,,,,
-01/11/17-20:49:28.523873 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A097C,0x569010D6,,0x1000,64,10,16560,1500,225300,,,,
-01/11/17-20:49:28.523877 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E3A0F24,0x569010D6,,0x1000,64,10,36010,788,20492,,,,
-01/11/17-20:49:28.523880 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A1204,,0x1F13,64,16,3952,52,53248,,,,
-01/11/17-20:49:28.524221 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A1204,0x569010D6,,0x1000,64,10,63499,1500,225300,,,,
-01/11/17-20:49:28.524225 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E3A17AC,0x569010D6,,0x1000,64,10,48918,948,184332,,,,
-01/11/17-20:49:28.524227 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A1B2C,,0x1F13,64,16,3953,52,53248,,,,
-01/11/17-20:49:28.524579 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A1B2C,0x569010D6,,0x1000,64,10,10124,1500,225300,,,,
-01/11/17-20:49:28.524582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E3A20D4,0x569010D6,,0x1000,64,10,60584,964,200716,,,,
-01/11/17-20:49:28.524585 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A2464,,0x1F13,64,16,3954,52,53248,,,,
-01/11/17-20:49:28.524889 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A2464,0x569010D6,,0x1000,64,10,30197,1500,225300,,,,
-01/11/17-20:49:28.524893 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E3A2A0C,0x569010D6,,0x1000,64,10,14240,756,249864,,,,
-01/11/17-20:49:28.524895 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569010D6,0x1E3A2CCC,,0x1F13,64,16,3955,52,53248,,,,
-01/11/17-20:49:28.525595 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2CCC,0x569010EE,,0xFFF,64,8,44823,52,53248,,,,
-01/11/17-20:49:28.525764 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2CCC,0x56901106,,0xFFF,64,8,60365,52,53248,,,,
-01/11/17-20:49:28.525893 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A2CCC,0x56901106,,0x1000,64,10,22857,92,94208,,,,
-01/11/17-20:49:28.526459 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2CF4,0x5690111E,,0xFFF,64,8,36935,52,53248,,,,
-01/11/17-20:49:28.526578 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2CF4,0x5690111E,,0x1000,64,10,1319,76,77824,,,,
-01/11/17-20:49:28.527102 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2D0C,0x56901166,,0xFFD,64,8,14691,52,53248,,,,
-01/11/17-20:49:28.527180 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2D0C,0x56901166,,0x1000,64,10,32628,76,77824,,,,
-01/11/17-20:49:28.540964 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A2D24,0x56901166,,0x1000,64,10,52141,92,94208,,,,
-01/11/17-20:49:28.541023 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901166,0x1E3A2D4C,,0x1F13,64,16,3960,52,53248,,,,
-01/11/17-20:49:28.541279 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2D4C,0x5690117E,,0xFFF,64,8,49024,52,53248,,,,
-01/11/17-20:49:28.541440 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E3A2D4C,0x5690117E,,0x1000,64,10,64071,556,45064,,,,
-01/11/17-20:49:28.542767 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x569011C6,,0xFFD,64,8,20305,52,53248,,,,
-01/11/17-20:49:28.568085 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x569011FE,,0xFFE,64,8,35412,52,53248,,,,
-01/11/17-20:49:28.618582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x56901236,,0xFFE,64,8,60632,52,53248,,,,
-01/11/17-20:49:28.626337 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F44,0x56901296,,0xFFD,64,8,20453,52,53248,,,,
-01/11/17-20:49:28.626346 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2F44,0x56901296,,0x1000,64,10,3210,76,77824,,,,
-01/11/17-20:49:28.671946 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901296,0x1E3A2F5C,,0x1F13,64,16,3966,52,53248,,,,
-01/11/17-20:49:38.644842 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A2F5C,0x56901296,,0x1000,64,10,35540,76,77824,,,,
-01/11/17-20:49:38.644865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901296,0x1E3A2F74,,0x1F13,64,16,3967,52,53248,,,,
-01/11/17-20:49:38.645301 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A2F74,0x569012AE,,0xFFF,64,8,38915,52,53248,,,,
-01/11/17-20:49:38.645410 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E3A2F74,0x569012AE,,0x1000,64,10,49946,204,208896,,,,
-01/11/17-20:49:38.655935 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A300C,0x569012F6,,0xFFD,64,8,49982,52,53248,,,,
-01/11/17-20:49:38.678474 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A300C,0x5690135E,,0xFFC,64,8,32965,52,53248,,,,
-01/11/17-20:49:38.678482 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A300C,0x5690138E,,0xFFE,64,8,24815,52,53248,,,,
-01/11/17-20:49:38.678626 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A300C,0x5690138E,,0x1000,64,10,26150,76,77824,,,,
-01/11/17-20:49:38.698531 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A3024,0x5690138E,,0x1000,64,10,57225,92,94208,,,,
-01/11/17-20:49:38.698676 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690138E,0x1E3A304C,,0x1F13,64,16,3972,52,53248,,,,
-01/11/17-20:49:38.698875 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A304C,0x569013A6,,0xFFF,64,8,22084,52,53248,,,,
-01/11/17-20:49:38.698882 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E3A304C,0x569013A6,,0x1000,64,10,64622,116,118784,,,,
-01/11/17-20:49:38.699881 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A308C,0x569013D6,,0xFFE,64,8,44450,52,53248,,,,
-01/11/17-20:49:38.700138 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A308C,0x569013D6,,0x1000,64,10,42900,92,94208,,,,
-01/11/17-20:49:38.710613 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A30B4,0x569013EE,,0xFFF,64,8,64776,52,53248,,,,
-01/11/17-20:49:38.710768 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A30B4,0x569013EE,,0x1000,64,10,19867,92,94208,,,,
-01/11/17-20:49:38.711060 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A30DC,0x56901406,,0xFFF,64,8,52782,52,53248,,,,
-01/11/17-20:49:38.711262 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A30DC,0x56901406,,0x1000,64,10,10692,92,94208,,,,
-01/11/17-20:49:38.711505 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A3104,0x5690141E,,0xFFF,64,8,8814,52,53248,,,,
-01/11/17-20:49:38.717620 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A3104,0x5690141E,,0x1000,64,10,62092,92,94208,,,,
-01/11/17-20:49:38.718189 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A312C,0x56901436,,0xFFF,64,8,42105,52,53248,,,,
-01/11/17-20:49:38.718603 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A312C,0x56901436,,0x1000,64,10,64933,1500,225300,,,,
-01/11/17-20:49:38.718608 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E3A36D4,0x56901436,,0x1000,64,10,26575,1060,36880,,,,
-01/11/17-20:49:38.718612 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A3AC4,,0x1F13,64,16,3979,52,53248,,,,
-01/11/17-20:49:38.718937 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A3AC4,0x56901436,,0x1000,64,10,43647,1500,225300,,,,
-01/11/17-20:49:38.718952 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A406C,0x56901436,,0x1000,64,10,31758,1500,225300,,,,
-01/11/17-20:49:38.718955 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A4614,,0x1F13,64,16,3980,52,53248,,,,
-01/11/17-20:49:38.718963 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4A,***AP***,0x1E3A4614,0x56901436,,0x1000,64,10,22512,60,61440,,,,
-01/11/17-20:49:38.719298 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A461C,0x56901436,,0x1000,64,10,21591,1500,225300,,,,
-01/11/17-20:49:38.719302 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A4BC4,,0x1F13,64,16,3981,52,53248,,,,
-01/11/17-20:49:38.719322 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E3A4BC4,0x56901436,,0x1000,64,10,59325,644,135176,,,,
-01/11/17-20:49:38.719683 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A4E14,0x56901436,,0x1000,64,10,60528,1500,225300,,,,
-01/11/17-20:49:38.719689 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A53BC,,0x1F13,64,16,3982,52,53248,,,,
-01/11/17-20:49:38.719707 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E3A53BC,0x56901436,,0x1000,64,10,51009,900,135180,,,,
-01/11/17-20:49:38.719709 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A570C,0x56901436,,0x1000,64,10,10473,92,94208,,,,
-01/11/17-20:49:38.719789 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901436,0x1E3A5734,,0x1F13,64,16,3983,52,53248,,,,
-01/11/17-20:49:38.720104 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A5734,0x5690144E,,0xFFF,64,8,62478,52,53248,,,,
-01/11/17-20:49:38.720549 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A5734,0x5690144E,,0x1000,64,10,24986,1500,225300,,,,
-01/11/17-20:49:38.720564 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E3A5CDC,0x5690144E,,0x1000,64,10,53439,292,36868,,,,
-01/11/17-20:49:38.720567 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A5DCC,,0x1F13,64,16,3985,52,53248,,,,
-01/11/17-20:49:38.720952 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A5DCC,0x5690144E,,0x1000,64,10,59946,1500,225300,,,,
-01/11/17-20:49:38.720973 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E3A6374,0x5690144E,,0x1000,64,10,27663,468,217092,,,,
-01/11/17-20:49:38.720977 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A6514,,0x1F13,64,16,3986,52,53248,,,,
-01/11/17-20:49:38.721423 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A6514,0x5690144E,,0x1000,64,10,6032,1500,225300,,,,
-01/11/17-20:49:38.721430 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E3A6ABC,0x5690144E,,0x1000,64,10,21413,164,167936,,,,
-01/11/17-20:49:38.721433 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A6B2C,,0x1F13,64,16,3987,52,53248,,,,
-01/11/17-20:49:38.721836 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A6B2C,0x5690144E,,0x1000,64,10,28700,1500,225300,,,,
-01/11/17-20:49:38.721841 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E3A70D4,0x5690144E,,0x1000,64,10,11267,788,20492,,,,
-01/11/17-20:49:38.721844 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A73B4,,0x1F13,64,16,3988,52,53248,,,,
-01/11/17-20:49:38.722366 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A73B4,0x5690144E,,0x1000,64,10,23915,1500,225300,,,,
-01/11/17-20:49:38.722373 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E3A795C,0x5690144E,,0x1000,64,10,22812,948,184332,,,,
-01/11/17-20:49:38.722377 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690144E,0x1E3A7CDC,,0x1F13,64,16,3989,52,53248,,,,
-01/11/17-20:49:38.722666 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A7CDC,0x5690144E,,0x1000,64,10,56330,1500,225300,,,,
-01/11/17-20:49:38.722669 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E3A8284,0x5690144E,,0x1000,64,10,9418,964,200716,,,,
-01/11/17-20:49:38.722672 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8614,0x56901466,,0xFFF,64,8,54552,52,53248,,,,
-01/11/17-20:49:38.722709 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901466,0x1E3A8614,,0x1F13,64,16,3991,52,53248,,,,
-01/11/17-20:49:38.723188 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A8614,0x56901466,,0x1000,64,10,36476,1500,225300,,,,
-01/11/17-20:49:38.723194 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E3A8BBC,0x56901466,,0x1000,64,10,5160,756,249864,,,,
-01/11/17-20:49:38.723197 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901466,0x1E3A8E7C,,0x1F13,64,16,3992,52,53248,,,,
-01/11/17-20:49:38.723727 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8E7C,0x5690147E,,0xFFF,64,8,58139,52,53248,,,,
-01/11/17-20:49:38.723829 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A8E7C,0x5690147E,,0x1000,64,10,13602,92,94208,,,,
-01/11/17-20:49:38.724393 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8EA4,0x56901496,,0xFFF,64,8,7887,52,53248,,,,
-01/11/17-20:49:38.724427 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A8EA4,0x56901496,,0x1000,64,10,26953,76,77824,,,,
-01/11/17-20:49:38.725021 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8EBC,0x569014DE,,0xFFD,64,8,9083,52,53248,,,,
-01/11/17-20:49:38.725028 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A8EBC,0x569014DE,,0x1000,64,10,25776,76,77824,,,,
-01/11/17-20:49:38.739469 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A8ED4,0x569014DE,,0x1000,64,10,30246,92,94208,,,,
-01/11/17-20:49:38.739600 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569014DE,0x1E3A8EFC,,0x1F13,64,16,3996,52,53248,,,,
-01/11/17-20:49:38.739910 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A8EFC,0x569014F6,,0xFFF,64,8,41919,52,53248,,,,
-01/11/17-20:49:38.740117 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E3A8EFC,0x569014F6,,0x1000,64,10,37312,556,45064,,,,
-01/11/17-20:49:38.741314 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x5690153E,,0xFFD,64,8,17721,52,53248,,,,
-01/11/17-20:49:38.771156 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x56901576,,0xFFE,64,8,55944,52,53248,,,,
-01/11/17-20:49:38.829875 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x569015BE,,0xFFD,64,8,4622,52,53248,,,,
-01/11/17-20:49:38.838484 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A90F4,0x5690161E,,0xFFD,64,8,12690,52,53248,,,,
-01/11/17-20:49:38.838557 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A90F4,0x5690161E,,0x1000,64,10,6352,76,77824,,,,
-01/11/17-20:49:38.864196 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A910C,0x5690161E,,0x1000,64,10,42677,76,77824,,,,
-01/11/17-20:49:38.864547 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690161E,0x1E3A9124,,0x1F13,64,16,4002,52,53248,,,,
-01/11/17-20:49:38.864823 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A9124,0x56901636,,0xFFF,64,8,62716,52,53248,,,,
-01/11/17-20:49:38.864973 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xDA,***AP***,0x1E3A9124,0x56901636,,0x1000,64,10,18619,204,208896,,,,
-01/11/17-20:49:38.866516 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91BC,0x5690167E,,0xFFD,64,8,13402,52,53248,,,,
-01/11/17-20:49:38.884253 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91BC,0x569016E6,,0xFFC,64,8,23445,52,53248,,,,
-01/11/17-20:49:38.884262 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91BC,0x56901716,,0xFFE,64,8,15998,52,53248,,,,
-01/11/17-20:49:38.884421 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3A91BC,0x56901716,,0x1000,64,10,37876,76,77824,,,,
-01/11/17-20:49:38.907086 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A91D4,0x56901716,,0x1000,64,10,41998,92,94208,,,,
-01/11/17-20:49:38.907273 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901716,0x1E3A91FC,,0x1F13,64,16,4007,52,53248,,,,
-01/11/17-20:49:38.907498 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A91FC,0x5690172E,,0xFFF,64,8,2031,52,53248,,,,
-01/11/17-20:49:38.907674 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x82,***AP***,0x1E3A91FC,0x5690172E,,0x1000,64,10,65433,116,118784,,,,
-01/11/17-20:49:38.909097 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A923C,0x5690175E,,0xFFE,64,8,41174,52,53248,,,,
-01/11/17-20:49:38.909108 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A923C,0x5690175E,,0x1000,64,10,28977,92,94208,,,,
-01/11/17-20:49:38.950050 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x5690175E,0x1E3A9264,,0x1F13,64,16,4010,52,53248,,,,
-01/11/17-20:49:38.952099 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A9264,0x56901776,,0xFFF,64,8,20145,52,53248,,,,
-01/11/17-20:49:38.952254 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A9264,0x56901776,,0x1000,64,10,1604,92,94208,,,,
-01/11/17-20:49:38.952898 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901776,0x1E3A928C,,0x1F13,64,16,4012,52,53248,,,,
-01/11/17-20:49:38.955697 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A928C,0x5690178E,,0xFFF,64,8,29670,52,53248,,,,
-01/11/17-20:49:38.955918 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A928C,0x5690178E,,0x1000,64,10,41295,92,94208,,,,
-01/11/17-20:49:38.957143 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A92B4,0x569017A6,,0xFFF,64,8,53853,52,53248,,,,
-01/11/17-20:49:38.965865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3A92B4,0x569017A6,,0x1000,64,10,34049,92,94208,,,,
-01/11/17-20:49:38.969753 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3A92DC,0x569017BE,,0xFFF,64,8,18267,52,53248,,,,
-01/11/17-20:49:38.970498 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A92DC,0x569017BE,,0x1000,64,10,33394,1500,225300,,,,
-01/11/17-20:49:38.970508 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x432,***AP***,0x1E3A9884,0x569017BE,,0x1000,64,10,24966,1060,36880,,,,
-01/11/17-20:49:38.970511 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3A9C74,,0x1F13,64,16,4016,52,53248,,,,
-01/11/17-20:49:38.971108 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3A9C74,0x569017BE,,0x1000,64,10,2345,1500,225300,,,,
-01/11/17-20:49:38.971119 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AA21C,0x569017BE,,0x1000,64,10,63264,1500,225300,,,,
-01/11/17-20:49:38.971123 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AA7C4,,0x1F13,64,16,4017,52,53248,,,,
-01/11/17-20:49:38.971144 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AA7C4,0x569017BE,,0x1000,64,10,38086,76,77824,,,,
-01/11/17-20:49:38.971566 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AA7DC,0x569017BE,,0x1000,64,10,15052,1500,225300,,,,
-01/11/17-20:49:38.971575 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AAD84,,0x1F13,64,16,4018,52,53248,,,,
-01/11/17-20:49:38.971602 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x292,***AP***,0x1E3AAD84,0x569017BE,,0x1000,64,10,23674,644,135176,,,,
-01/11/17-20:49:38.971871 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AAFD4,0x569017BE,,0x1000,64,10,48549,1500,225300,,,,
-01/11/17-20:49:38.971876 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AB57C,,0x1F13,64,16,4019,52,53248,,,,
-01/11/17-20:49:38.971888 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x392,***AP***,0x1E3AB57C,0x569017BE,,0x1000,64,10,2232,900,135180,,,,
-01/11/17-20:49:38.971890 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3AB8CC,0x569017BE,,0x1000,64,10,11241,92,94208,,,,
-01/11/17-20:49:38.974036 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017BE,0x1E3AB8F4,,0x1F13,64,16,4020,52,53248,,,,
-01/11/17-20:49:38.974671 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AB8F4,0x569017D6,,0xFFF,64,8,57358,52,53248,,,,
-01/11/17-20:49:38.974986 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AB8F4,0x569017D6,,0x1000,64,10,60275,1500,225300,,,,
-01/11/17-20:49:38.974992 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x132,***AP***,0x1E3ABE9C,0x569017D6,,0x1000,64,10,29505,292,36868,,,,
-01/11/17-20:49:38.974995 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3ABF8C,,0x1F13,64,16,4022,52,53248,,,,
-01/11/17-20:49:38.975462 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3ABF8C,0x569017D6,,0x1000,64,10,62499,1500,225300,,,,
-01/11/17-20:49:38.975466 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E2,***AP***,0x1E3AC534,0x569017D6,,0x1000,64,10,47337,468,217092,,,,
-01/11/17-20:49:38.975469 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3AC6D4,,0x1F13,64,16,4023,52,53248,,,,
-01/11/17-20:49:38.975865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AC6D4,0x569017D6,,0x1000,64,10,33384,1500,225300,,,,
-01/11/17-20:49:38.975870 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0xB2,***AP***,0x1E3ACC7C,0x569017D6,,0x1000,64,10,34219,164,167936,,,,
-01/11/17-20:49:38.975873 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3ACCEC,,0x1F13,64,16,4024,52,53248,,,,
-01/11/17-20:49:38.976279 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3ACCEC,0x569017D6,,0x1000,64,10,39935,1500,225300,,,,
-01/11/17-20:49:38.976284 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x322,***AP***,0x1E3AD294,0x569017D6,,0x1000,64,10,65405,788,20492,,,,
-01/11/17-20:49:38.976287 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3AD574,,0x1F13,64,16,4025,52,53248,,,,
-01/11/17-20:49:38.976672 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AD574,0x569017D6,,0x1000,64,10,47260,1500,225300,,,,
-01/11/17-20:49:38.976677 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3C2,***AP***,0x1E3ADB1C,0x569017D6,,0x1000,64,10,7038,948,184332,,,,
-01/11/17-20:49:38.976680 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3ADE9C,,0x1F13,64,16,4026,52,53248,,,,
-01/11/17-20:49:38.977244 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3ADE9C,0x569017D6,,0x1000,64,10,57825,1500,225300,,,,
-01/11/17-20:49:38.977252 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x3D2,***AP***,0x1E3AE444,0x569017D6,,0x1000,64,10,14750,964,200716,,,,
-01/11/17-20:49:38.977256 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017D6,0x1E3AE7D4,,0x1F13,64,16,4027,52,53248,,,,
-01/11/17-20:49:38.977582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AE7D4,0x569017EE,,0xFFF,64,8,40965,52,53248,,,,
-01/11/17-20:49:38.977740 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5EA,***A****,0x1E3AE7D4,0x569017EE,,0x1000,64,10,25235,1500,225300,,,,
-01/11/17-20:49:38.977746 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x302,***AP***,0x1E3AED7C,0x569017EE,,0x1000,64,10,37899,756,249864,,,,
-01/11/17-20:49:38.977748 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569017EE,0x1E3AF03C,,0x1F13,64,16,4029,52,53248,,,,
-01/11/17-20:49:38.978306 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF03C,0x56901806,,0xFFF,64,8,5277,52,53248,,,,
-01/11/17-20:49:38.978429 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3AF03C,0x56901806,,0x1000,64,10,39595,92,94208,,,,
-01/11/17-20:49:38.978660 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF064,0x5690181E,,0xFFF,64,8,18434,52,53248,,,,
-01/11/17-20:49:38.978743 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AF064,0x5690181E,,0x1000,64,10,59548,76,77824,,,,
-01/11/17-20:49:38.979085 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF07C,0x56901866,,0xFFD,64,8,64094,52,53248,,,,
-01/11/17-20:49:38.979195 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AF07C,0x56901866,,0x1000,64,10,27814,76,77824,,,,
-01/11/17-20:49:38.996191 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x6A,***AP***,0x1E3AF094,0x56901866,,0x1000,64,10,58142,92,94208,,,,
-01/11/17-20:49:38.996344 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x56901866,0x1E3AF0BC,,0x1F13,64,16,4033,52,53248,,,,
-01/11/17-20:49:38.996582 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF0BC,0x5690187E,,0xFFF,64,8,40934,52,53248,,,,
-01/11/17-20:49:38.996788 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x23A,***AP***,0x1E3AF0BC,0x5690187E,,0x1000,64,10,41880,556,45064,,,,
-01/11/17-20:49:38.998362 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x569018C6,,0xFFD,64,8,46597,52,53248,,,,
-01/11/17-20:49:39.029755 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x569018FE,,0xFFE,64,8,50704,52,53248,,,,
-01/11/17-20:49:39.090878 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x56901946,,0xFFD,64,8,58336,52,53248,,,,
-01/11/17-20:49:39.103926 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x1E3AF2B4,0x569019A6,,0xFFD,64,8,6076,52,53248,,,,
-01/11/17-20:49:39.103935 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,49581,192.168.66.121,22,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x5A,***AP***,0x1E3AF2B4,0x569019A6,,0x1000,64,10,7481,76,77824,,,,
-01/11/17-20:49:39.144528 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,22,192.168.66.1,49581,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x569019A6,0x1E3AF2CC,,0x1F13,64,16,4039,52,53248,,,,
-01/11/17-20:49:41.685007 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,12****S*,0x89DE3802,0x0,,0xFFFF,64,0,37837,64,65536,,,,
-01/11/17-20:49:41.685041 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50181,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,*2*A**S*,0x412FE1F9,0x89DE3803,,0x3890,64,0,0,60,61440,,,,
-01/11/17-20:49:41.685270 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3803,0x412FE1FA,,0x1015,64,0,45378,52,53248,,,,
-01/11/17-20:49:41.685291 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x203,***AP***,0x89DE3803,0x412FE1FA,,0x1015,64,2,31618,501,250884,,,,
-01/11/17-20:49:41.685297 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50181,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x412FE1FA,0x89DE39C4,,0x7A,64,0,35921,52,53248,,,,
-01/11/17-20:49:41.702785 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE39C4,0x412FEAB5,,0xFE7,64,0,31331,52,53248,,,,
-01/11/17-20:49:41.708304 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1E0,***AP***,0x89DE39C4,0x412FEAB5,,0x1000,64,2,55934,466,215044,,,,
-01/11/17-20:49:41.708865 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,12****S*,0xE53A56EE,0x0,,0xFFFF,64,0,35683,64,65536,,,,
-01/11/17-20:49:41.708880 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50182,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,*2*A**S*,0x18588B4A,0xE53A56EF,,0x3890,64,0,0,60,61440,,,,
-01/11/17-20:49:41.709029 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A56EF,0x18588B4B,,0x1015,64,0,32055,52,53248,,,,
-01/11/17-20:49:41.709146 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1DD,***AP***,0xE53A56EF,0x18588B4B,,0x1015,64,2,10852,463,211972,,,,
-01/11/17-20:49:41.709154 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50182,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x18588B4B,0xE53A588A,,0x7A,64,0,59016,52,53248,,,,
-01/11/17-20:49:41.710292 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,12****S*,0x8365AF51,0x0,,0xFFFF,64,0,7652,64,65536,,,,
-01/11/17-20:49:41.710309 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50183,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,*2*A**S*,0x8DE9822F,0x8365AF52,,0x3890,64,0,0,60,61440,,,,
-01/11/17-20:49:41.710470 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365AF52,0x8DE98230,,0x1015,64,0,60499,52,53248,,,,
-01/11/17-20:49:41.710664 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1D0,***AP***,0x8365AF52,0x8DE98230,,0x1015,64,2,49526,450,198660,,,,
-01/11/17-20:49:41.710674 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50183,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x8DE98230,0x8365B0E0,,0x7A,64,0,61429,52,53248,,,,
-01/11/17-20:49:41.713117 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x4E,******S*,0x39818AB4,0x0,,0xFFFF,64,0,31071,64,65536,,,,
-01/11/17-20:49:41.713131 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50184,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x4A,***A**S*,0xA874A3E9,0x39818AB5,,0x3890,64,0,0,60,61440,,,,
-01/11/17-20:49:41.713287 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818AB5,0xA874A3EA,,0x1015,64,0,63436,52,53248,,,,
-01/11/17-20:49:41.713427 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x1CD,***AP***,0x39818AB5,0xA874A3EA,,0x1015,64,0,50310,447,195588,,,,
-01/11/17-20:49:41.713435 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50184,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0xA874A3EA,0x39818C40,,0x7A,64,0,52781,52,53248,,,,
-01/11/17-20:49:41.735024 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE98D80,,0xFBB,64,0,60492,52,53248,,,,
-01/11/17-20:49:41.735121 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE998D0,,0xFA5,64,0,27946,52,53248,,,,
-01/11/17-20:49:41.735230 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9A420,,0xF4B,64,0,12533,52,53248,,,,
-01/11/17-20:49:41.735368 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9AF70,,0xEF0,64,0,61911,52,53248,,,,
-01/11/17-20:49:41.735487 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9BAC0,,0xE96,64,0,32463,52,53248,,,,
-01/11/17-20:49:41.735764 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9C610,,0xE3B,64,0,41222,52,53248,,,,
-01/11/17-20:49:41.735771 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9C610,,0x1000,64,0,17505,52,53248,,,,
-01/11/17-20:49:41.735934 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9D160,,0xFA5,64,0,6378,52,53248,,,,
-01/11/17-20:49:41.735939 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9DCB0,,0xF4B,64,0,50978,52,53248,,,,
-01/11/17-20:49:41.736031 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9E800,,0xEF0,64,0,41992,52,53248,,,,
-01/11/17-20:49:41.736034 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9F350,,0xE96,64,0,40068,52,53248,,,,
-01/11/17-20:49:41.736039 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DE9FEA0,,0xE3B,64,0,4465,52,53248,,,,
-01/11/17-20:49:41.736130 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA0365,,0xE15,64,0,64175,52,53248,,,,
-01/11/17-20:49:41.736242 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA0365,,0x1000,64,0,60532,52,53248,,,,
-01/11/17-20:49:41.736790 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA0798,,0xFDE,64,0,1924,52,53248,,,,
-01/11/17-20:49:41.740496 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA079E,,0xFFF,64,0,12835,52,53248,,,,
-01/11/17-20:49:41.747598 ,1,999158,0,"'snort test alert'",TCP,192.168.66.121,8080,192.168.66.1,50181,08:00:27:E8:B0:7A,0A:00:27:00:00:00,0x42,***A****,0x412FEAB5,0x89DE3B62,,0x82,64,0,35924,52,53248,,,,
-01/11/17-20:49:41.757549 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858969B,,0xFBB,64,0,20679,52,53248,,,,
-01/11/17-20:49:41.758290 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858A1EB,,0xFA5,64,0,44022,52,53248,,,,
-01/11/17-20:49:41.758399 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858AD3B,,0xF4B,64,0,28875,52,53248,,,,
-01/11/17-20:49:41.758404 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858B88B,,0xEF0,64,0,43317,52,53248,,,,
-01/11/17-20:49:41.758405 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858C3DB,,0xE96,64,0,55975,52,53248,,,,
-01/11/17-20:49:41.758406 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858C3DB,,0x1000,64,0,19782,52,53248,,,,
-01/11/17-20:49:41.758408 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858CF2B,,0xFA5,64,0,63139,52,53248,,,,
-01/11/17-20:49:41.759044 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858CF2B,,0x1000,64,0,49835,52,53248,,,,
-01/11/17-20:49:41.771340 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x412FF605,,0xFA5,64,0,44665,52,53248,,,,
-01/11/17-20:49:41.773636 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41300155,,0xF4B,64,0,6905,52,53248,,,,
-01/11/17-20:49:41.773648 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41300CA5,,0xEF0,64,0,61971,52,53248,,,,
-01/11/17-20:49:41.773653 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x413017F5,,0xE96,64,0,31949,52,53248,,,,
-01/11/17-20:49:41.773656 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302345,,0xE3B,64,0,48816,52,53248,,,,
-01/11/17-20:49:41.773658 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302345,,0xEBB,64,0,30870,52,53248,,,,
-01/11/17-20:49:41.773660 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302345,,0x1000,64,0,8585,52,53248,,,,
-01/11/17-20:49:41.773662 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302E95,,0xFA5,64,0,61708,52,53248,,,,
-01/11/17-20:49:41.773664 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41302E95,,0x1000,64,0,38538,52,53248,,,,
-01/11/17-20:49:41.777099 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858DA7B,,0xFA5,64,0,23510,52,53248,,,,
-01/11/17-20:49:41.777115 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858E5CB,,0xF4B,64,0,16857,52,53248,,,,
-01/11/17-20:49:41.777119 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858F11B,,0xEF0,64,0,34551,52,53248,,,,
-01/11/17-20:49:41.777122 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x1858FC6B,,0xE96,64,0,63082,52,53248,,,,
-01/11/17-20:49:41.777127 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185907BB,,0xE3B,64,0,37755,52,53248,,,,
-01/11/17-20:49:41.777129 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590C70,,0xE15,64,0,42363,52,53248,,,,
-01/11/17-20:49:41.777132 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590F7F,,0xDFD,64,0,8363,52,53248,,,,
-01/11/17-20:49:41.777134 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x413039E5,,0xFA5,64,0,11423,52,53248,,,,
-01/11/17-20:49:41.777248 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41304535,,0xF4B,64,0,53395,52,53248,,,,
-01/11/17-20:49:41.777254 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41305085,,0xEF0,64,0,42374,52,53248,,,,
-01/11/17-20:49:41.777256 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41305BD5,,0xE96,64,0,27546,52,53248,,,,
-01/11/17-20:49:41.777259 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41306725,,0xE3B,64,0,42745,52,53248,,,,
-01/11/17-20:49:41.777261 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41306BDA,,0xE15,64,0,64435,52,53248,,,,
-01/11/17-20:49:41.777264 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590F7F,,0x1000,64,0,43557,52,53248,,,,
-01/11/17-20:49:41.777266 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41306BDA,,0x1000,64,0,46958,52,53248,,,,
-01/11/17-20:49:41.778744 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x4130772A,,0xFA5,64,0,14812,52,53248,,,,
-01/11/17-20:49:41.778811 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41307BED,,0xF7F,64,0,16449,52,53248,,,,
-01/11/17-20:49:41.778817 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41307BED,,0x1000,64,0,32765,52,53248,,,,
-01/11/17-20:49:41.779807 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41307BF3,,0xFFF,64,0,61783,52,53248,,,,
-01/11/17-20:49:41.779816 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18590F85,,0xFFF,64,0,21799,52,53248,,,,
-01/11/17-20:49:41.780562 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874AF3A,,0xFBB,64,0,41668,52,53248,,,,
-01/11/17-20:49:41.780593 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874BA8A,,0xF60,64,0,44575,52,53248,,,,
-01/11/17-20:49:41.780611 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874BA8A,,0xFCB,64,0,60316,52,53248,,,,
-01/11/17-20:49:41.780614 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874C5DA,,0xF70,64,0,29140,52,53248,,,,
-01/11/17-20:49:41.781366 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874D12A,,0xF16,64,0,29289,52,53248,,,,
-01/11/17-20:49:41.781471 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874DC7A,,0xEBB,64,0,29562,52,53248,,,,
-01/11/17-20:49:41.781478 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874E7CA,,0xE61,64,0,48236,52,53248,,,,
-01/11/17-20:49:41.781624 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874F31A,,0xE06,64,0,39280,52,53248,,,,
-01/11/17-20:49:41.781630 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874FE6A,,0xDAC,64,0,8700,52,53248,,,,
-01/11/17-20:49:41.781633 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA874FE6A,,0x1000,64,0,10159,52,53248,,,,
-01/11/17-20:49:41.781924 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA87509BA,,0xFA5,64,0,11390,52,53248,,,,
-01/11/17-20:49:41.781933 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA87509BA,,0x1000,64,0,24685,52,53248,,,,
-01/11/17-20:49:41.781936 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875150A,,0xFA5,64,0,3366,52,53248,,,,
-01/11/17-20:49:41.781938 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875205A,,0xF4B,64,0,36802,52,53248,,,,
-01/11/17-20:49:41.781941 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875205A,,0x1000,64,0,13880,52,53248,,,,
-01/11/17-20:49:41.781943 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8752BAA,,0xFA5,64,0,27392,52,53248,,,,
-01/11/17-20:49:41.781945 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA875360B,,0xF52,64,0,28642,52,53248,,,,
-01/11/17-20:49:41.781949 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8753611,,0xF52,64,0,55533,52,53248,,,,
-01/11/17-20:49:41.782121 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8753611,,0x1000,64,0,43032,52,53248,,,,
-01/11/17-20:49:41.799072 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA12EE,,0xFA5,64,0,30116,52,53248,,,,
-01/11/17-20:49:41.799086 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA1E3E,,0xF4B,64,0,57836,52,53248,,,,
-01/11/17-20:49:41.799089 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41308743,,0xFA5,64,0,45743,52,53248,,,,
-01/11/17-20:49:41.799096 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309293,,0xF4B,64,0,19041,52,53248,,,,
-01/11/17-20:49:41.799099 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309BF3,,0xF00,64,0,56110,52,53248,,,,
-01/11/17-20:49:41.799101 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309F20,,0xEE6,64,0,14796,52,53248,,,,
-01/11/17-20:49:41.799103 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA279E,,0xF00,64,0,42649,52,53248,,,,
-01/11/17-20:49:41.799106 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA279E,,0x1000,64,0,13788,52,53248,,,,
-01/11/17-20:49:41.799107 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309F20,,0x1000,64,0,44776,52,53248,,,,
-01/11/17-20:49:41.799109 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50181,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x89DE3B62,0x41309F25,,0xFFF,64,0,2086,52,53248,,,,
-01/11/17-20:49:41.799832 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18591AD5,,0xFA5,64,0,64984,52,53248,,,,
-01/11/17-20:49:41.799898 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18591AD5,,0x1000,64,0,3933,52,53248,,,,
-01/11/17-20:49:41.799900 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592625,,0xFA5,64,0,16728,52,53248,,,,
-01/11/17-20:49:41.800244 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592F85,,0xF5A,64,0,16089,52,53248,,,,
-01/11/17-20:49:41.800252 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592F85,,0x1000,64,0,49727,52,53248,,,,
-01/11/17-20:49:41.800410 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18592F8D,,0xFFF,64,0,46722,52,53248,,,,
-01/11/17-20:49:41.800418 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA27A6,,0xFFF,64,0,44196,52,53248,,,,
-01/11/17-20:49:41.800513 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x18593ADD,,0xFA5,64,0,62171,52,53248,,,,
-01/11/17-20:49:41.800519 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185944C2,,0xF56,64,0,16322,52,53248,,,,
-01/11/17-20:49:41.800523 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185944C9,,0xF55,64,0,48908,52,53248,,,,
-01/11/17-20:49:41.800525 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA32F6,,0xFA5,64,0,41536,52,53248,,,,
-01/11/17-20:49:41.800559 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA3E46,,0xF4A,64,0,5801,52,53248,,,,
-01/11/17-20:49:41.800563 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50182,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0xE53A588A,0x185944C9,,0x1000,64,0,38774,52,53248,,,,
-01/11/17-20:49:41.800565 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA47A6,,0xEFF,64,0,15422,52,53248,,,,
-01/11/17-20:49:41.800567 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA47A6,,0x1000,64,0,5924,52,53248,,,,
-01/11/17-20:49:41.809759 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA47AD,,0xFFF,64,0,44728,52,53248,,,,
-01/11/17-20:49:41.809787 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50183,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x8365B0E0,0x8DEA5293,,0xFA8,64,0,19718,52,53248,,,,
-01/11/17-20:49:41.810740 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8754161,,0xFA5,64,0,7993,52,53248,,,,
-01/11/17-20:49:41.810762 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8754161,,0x1000,64,0,58879,52,53248,,,,
-01/11/17-20:49:41.810764 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8754CB1,,0xFA5,64,0,785,52,53248,,,,
-01/11/17-20:49:41.810767 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8755611,,0xF5A,64,0,54091,52,53248,,,,
-01/11/17-20:49:41.812571 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8755611,,0x1000,64,0,27089,52,53248,,,,
-01/11/17-20:49:41.812579 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8755619,,0xFFF,64,0,50244,52,53248,,,,
-01/11/17-20:49:41.812584 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8756169,,0xFA5,64,0,6997,52,53248,,,,
-01/11/17-20:49:41.812586 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8756169,,0x1000,64,0,45003,52,53248,,,,
-01/11/17-20:49:41.812587 ,1,999158,0,"'snort test alert'",TCP,192.168.66.1,50184,192.168.66.121,8080,0A:00:27:00:00:00,08:00:27:E8:B0:7A,0x42,***A****,0x39818C40,0xA8756CB9,,0xFA5,64,0,18800,52,53248,,,,
-01/11/17-20:49:4
<TRUNCATED>