You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/01/29 21:04:43 UTC

[30/30] incubator-metron git commit: METRON-21 Extend the ansible scripts to support vagrant

METRON-21 Extend the ansible scripts to support vagrant

No need to go and look for the latest version, which just slows down the deployment

Adding additional ignores for common Eclipse configs

Auto-magically launching the bro topology

Need to set JAVA_HOME using the Ansible var java_home

Snort alerts are being delivered to Kafka as CSV

Initial pass at adding the snort topology

Snort topology successfully shoveling messages to elasticsearch

Need to set JAVA_HOME using the Ansible var java_home

Added yaf source and topology

ALERTMODE needs to be empty for it not to complain when logging csv alerts.  No idea why.  Undocumented, trial-and-error, hack.

Turning off packet logging as not needed

added ability to load preconfigured Metron UI dashboard

Increased storm workers to 4

cleaned up yaf grok file


Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/54c8de38
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/54c8de38
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/54c8de38

Branch: refs/heads/master
Commit: 54c8de382250f3f281a6acba5833e199b00cebd4
Parents: e8b2ea6
Author: Nick Allen <ni...@nickallen.org>
Authored: Tue Jan 26 13:58:17 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Fri Jan 29 14:53:17 2016 -0500

----------------------------------------------------------------------
 .gitignore                                      |    3 +
 .../inventory/metron_example/group_vars/all     |    1 +
 .../inventory/singlenode-vagrant/group_vars/all |    3 +-
 deployment/roles/ambari_config/tasks/main.yml   |    2 +-
 .../roles/ambari_config/vars/single_node_vm.yml |    2 +
 .../roles/ambari_gather_facts/tasks/main.yml    |    6 +-
 deployment/roles/ansible-hdp-kafka/README.md    |   50 +
 .../roles/ansible-hdp-kafka/defaults/main.yml   |    2 +
 .../roles/ansible-hdp-kafka/handlers/main.yml   |    2 +
 .../ansible-hdp-kafka/meta/.galaxy_install_info |    1 +
 .../roles/ansible-hdp-kafka/meta/main.yml       |  147 +
 .../roles/ansible-hdp-kafka/tasks/main.yml      |   22 +
 .../roles/ansible-hdp-kafka/vars/main.yml       |    2 +
 deployment/roles/flume/tasks/main.yml           |    2 +-
 .../metron_streaming/files/yaf_index.template   |   36 +
 .../roles/metron_streaming/tasks/main.yml       |   26 +-
 deployment/roles/metron_streaming/vars/main.yml |    1 -
 deployment/roles/metron_ui/files/default.json   |  508 ++++
 deployment/roles/metron_ui/tasks/main.yml       |    6 +
 deployment/roles/snort/tasks/flume.yml          |   24 +-
 deployment/roles/snort/tasks/main.yml           |    3 +
 deployment/roles/snort/tasks/snort.yml          |   24 +-
 deployment/roles/yaf/files/start-yaf.sh         |    1 +
 deployment/roles/yaf/tasks/main.yml             |   76 +-
 deployment/roles/yaf/tasks/yaf.yml              |   33 +-
 deployment/roles/yaf/vars/main.yml              |    1 +
 .../parsing/parsers/BasicSnortParser.java       |  134 +
 .../metron/parsing/parsers/BasicYafParser.java  |  191 ++
 .../src/main/resources/patterns/yaf             |  113 +
 .../Metron_Configs/etc/env/config.properties    |   10 +-
 .../Metron_Configs/topologies/asa/remote.yaml   |    4 +-
 .../Metron_Configs/topologies/bro/remote.yaml   |    4 +-
 .../topologies/fireeye/remote.yaml              |    4 +-
 .../Metron_Configs/topologies/ise/remote.yaml   |    4 +-
 .../topologies/lancope/remote.yaml              |    4 +-
 .../topologies/paloalto/remote.yaml             |    4 +-
 .../Metron_Configs/topologies/pcap/remote.yaml  |    4 +-
 .../Metron_Configs/topologies/snort/local.yaml  |  188 ++
 .../Metron_Configs/topologies/snort/remote.yaml |  201 ++
 .../Metron_Configs/topologies/yaf/local.yaml    |  185 ++
 .../Metron_Configs/topologies/yaf/remote.yaml   |  199 ++
 .../src/main/resources/SampleInput/SnortOutput  |    1 +
 .../main/resources/SampleInput/YafExampleOutput | 2691 ++++++++++++++++++
 43 files changed, 4849 insertions(+), 76 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 4ab779f..f6c6cc5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,3 +13,6 @@ target
 .idea
 *.iml
 .DS_Store
+*.project
+*.classpath
+*.settings

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index 9686ddb..6f712d9 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -30,3 +30,4 @@ pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
 pycapa_home: "/opt/pycapa"
 pycapa_topic: "pcap"
 pcap_hbase_table: "pcap"
+snort_alert_csv_path: "/var/log/snort/alert.csv"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index f61be80..44d954b 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -31,4 +31,5 @@ bro_topic: "bro"
 pcap_hbase_table: "pcap_test"
 pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
 pycapa_home: "/opt/pycapa"
-pycapa_topic: "pcap"
\ No newline at end of file
+pycapa_topic: "pcap"
+snort_alert_csv_path: "/var/log/snort/alert.csv"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ambari_config/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/tasks/main.yml b/deployment/roles/ambari_config/tasks/main.yml
index 7b8864e..764399f 100644
--- a/deployment/roles/ambari_config/tasks/main.yml
+++ b/deployment/roles/ambari_config/tasks/main.yml
@@ -4,7 +4,7 @@
 - include_vars: "{{ cluster_type }}.yml"
 
 - name: Install python-requests
-  yum: name=python-requests state=latest
+  yum: name=python-requests
 
 - name: Create the ambari cluster
   ambari_cluster_state:

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml
index d26be12..07ca31b 100644
--- a/deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -32,6 +32,8 @@ configurations:
   - yarn-site:
       yarn.nodemanager.resource.memory-mb: 1024
       yarn.scheduler.maximum-allocation-mb: 1024
+  - storm-site:
+      supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
 
 blueprint:
   stack_name: HDP

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_gather_facts/tasks/main.yml b/deployment/roles/ambari_gather_facts/tasks/main.yml
index eb0b4ca..6f358ad 100644
--- a/deployment/roles/ambari_gather_facts/tasks/main.yml
+++ b/deployment/roles/ambari_gather_facts/tasks/main.yml
@@ -1,9 +1,7 @@
 ---
 # tasks file for ambari_gather_facts
 - name: Install pip
-  yum:
-    name: python-pip
-    state: latest
+  yum: name=python-pip
 
 - name: Install python httplib2 dependency
   pip:
@@ -142,4 +140,4 @@
           zookeeper_url = {{ zookeeper_url }},
           kafka_broker_port = {{ kafka_broker_port }},
           kafka_broker_hosts = {{ kafka_broker_hosts }},
-          kafka_broker_url = {{ kafka_broker_url }}"
\ No newline at end of file
+          kafka_broker_url = {{ kafka_broker_url }}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/README.md b/deployment/roles/ansible-hdp-kafka/README.md
new file mode 100644
index 0000000..e0e3466
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/README.md
@@ -0,0 +1,50 @@
+Kafka @ Hortonworks Data Platform
+=================================
+
+An Ansible Role that installs a single-node Kafka broker from the Hortonworks Data Platform distribution.  Zookeeper is also installed to support the Kafka broker.  
+
+Getting Started
+---------------
+
+Add to your playbook's requirements.yml:
+
+```
+- src: https://github.com/nickwallen/ansible-hdp-kafka
+```
+
+and then run:
+
+```
+ansible-galaxy install -r requirements.yml --ignore-errors
+```
+
+Requirements
+------------
+
+None.
+
+Role Variables
+--------------
+
+
+Dependencies
+------------
+
+None.
+
+Example Playbook
+----------------
+
+    - hosts: kafka*
+      roles:
+         - { role: ansible-hdp-kafka, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+Nick Allen

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/defaults/main.yml b/deployment/roles/ansible-hdp-kafka/defaults/main.yml
new file mode 100644
index 0000000..c7837fc
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/defaults/main.yml
@@ -0,0 +1,2 @@
+---
+# defaults file for .

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/handlers/main.yml b/deployment/roles/ansible-hdp-kafka/handlers/main.yml
new file mode 100644
index 0000000..050cdd1
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/handlers/main.yml
@@ -0,0 +1,2 @@
+---
+# handlers file for .

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/meta/.galaxy_install_info
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/meta/.galaxy_install_info b/deployment/roles/ansible-hdp-kafka/meta/.galaxy_install_info
new file mode 100644
index 0000000..c0c2c3c
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/meta/.galaxy_install_info
@@ -0,0 +1 @@
+{install_date: 'Wed Jan 27 14:27:44 2016', version: ''}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/meta/main.yml b/deployment/roles/ansible-hdp-kafka/meta/main.yml
new file mode 100644
index 0000000..888088e
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/meta/main.yml
@@ -0,0 +1,147 @@
+---
+galaxy_info:
+  author: your name
+  description: 
+  company: your company (optional)
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: http://example.com/issue/tracker
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: license (GPLv2, CC-BY, etc)
+  min_ansible_version: 1.2
+  #
+  # Below are all platforms currently available. Just uncomment
+  # the ones that apply to your role. If you don't see your 
+  # platform on this list, let us know and we'll get it added!
+  #
+  #platforms:
+  #- name: EL
+  #  versions:
+  #  - all
+  #  - 5
+  #  - 6
+  #  - 7
+  #- name: GenericUNIX
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Solaris
+  #  versions:
+  #  - all
+  #  - 10
+  #  - 11.0
+  #  - 11.1
+  #  - 11.2
+  #  - 11.3
+  #- name: Fedora
+  #  versions:
+  #  - all
+  #  - 16
+  #  - 17
+  #  - 18
+  #  - 19
+  #  - 20
+  #  - 21
+  #  - 22
+  #- name: Windows
+  #  versions:
+  #  - all
+  #  - 2012R2
+  #- name: SmartOS
+  #  versions:
+  #  - all
+  #  - any
+  #- name: opensuse
+  #  versions:
+  #  - all
+  #  - 12.1
+  #  - 12.2
+  #  - 12.3
+  #  - 13.1
+  #  - 13.2
+  #- name: Amazon
+  #  versions:
+  #  - all
+  #  - 2013.03
+  #  - 2013.09
+  #- name: GenericBSD
+  #  versions:
+  #  - all
+  #  - any
+  #- name: FreeBSD
+  #  versions:
+  #  - all
+  #  - 8.0
+  #  - 8.1
+  #  - 8.2
+  #  - 8.3
+  #  - 8.4
+  #  - 9.0
+  #  - 9.1
+  #  - 9.1
+  #  - 9.2
+  #- name: Ubuntu
+  #  versions:
+  #  - all
+  #  - lucid
+  #  - maverick
+  #  - natty
+  #  - oneiric
+  #  - precise
+  #  - quantal
+  #  - raring
+  #  - saucy
+  #  - trusty
+  #  - utopic
+  #  - vivid
+  #- name: SLES
+  #  versions:
+  #  - all
+  #  - 10SP3
+  #  - 10SP4
+  #  - 11
+  #  - 11SP1
+  #  - 11SP2
+  #  - 11SP3
+  #- name: GenericLinux
+  #  versions:
+  #  - all
+  #  - any
+  #- name: Debian
+  #  versions:
+  #  - all
+  #  - etch
+  #  - jessie
+  #  - lenny
+  #  - squeeze
+  #  - wheezy
+  #
+  # Below are all categories currently available. Just as with
+  # the platforms above, uncomment those that apply to your role.
+  #
+  #categories:
+  #- cloud
+  #- cloud:ec2
+  #- cloud:gce
+  #- cloud:rax
+  #- clustering
+  #- database
+  #- database:nosql
+  #- database:sql
+  #- development
+  #- monitoring
+  #- networking
+  #- packaging
+  #- system
+  #- web
+dependencies: []
+  # List your role dependencies here, one per line.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/tasks/main.yml b/deployment/roles/ansible-hdp-kafka/tasks/main.yml
new file mode 100644
index 0000000..c77b91b
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/tasks/main.yml
@@ -0,0 +1,22 @@
+---
+- name: Install prerequisites
+  yum: name=libselinux-python state=latest
+
+- name: Retrieve HDP repository definition
+  get_url:
+    url: "{{ hdp_repo_def }}"
+    dest: /etc/yum.repos.d/hdp.repo
+    mode: 0644
+
+- name: Install kafka
+  yum: name={{item}}
+  with_items:
+    - java-1.8.0-openjdk
+    - kafka
+    - zookeeper-server
+
+- name: Start zookeeper
+  shell: /usr/hdp/current/zookeeper-server/bin/zookeeper-server start
+
+- name: Start kafka
+  shell: /usr/hdp/current/kafka-broker/bin/kafka start

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/ansible-hdp-kafka/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ansible-hdp-kafka/vars/main.yml b/deployment/roles/ansible-hdp-kafka/vars/main.yml
new file mode 100644
index 0000000..d6af4b7
--- /dev/null
+++ b/deployment/roles/ansible-hdp-kafka/vars/main.yml
@@ -0,0 +1,2 @@
+---
+hdp_repo_def: http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.2.0/hdp.repo

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/flume/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/flume/tasks/main.yml b/deployment/roles/flume/tasks/main.yml
index 9d7f2ac..9239280 100644
--- a/deployment/roles/flume/tasks/main.yml
+++ b/deployment/roles/flume/tasks/main.yml
@@ -21,7 +21,7 @@
     line="{{ item.line }}"
     state=present
   with_items:
-    - { regexp: '^.*export JAVA_HOME=.*$', line: 'export JAVA_HOME=/usr/jdk64/jdk1.8.0_40/' }
+    - { regexp: '^.*export JAVA_HOME=.*$', line: 'export JAVA_HOME={{ java_home }}' }
 
 - name: Create flume service
   shell: "{{item}}"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/metron_streaming/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/files/yaf_index.template b/deployment/roles/metron_streaming/files/yaf_index.template
new file mode 100644
index 0000000..c8c1702
--- /dev/null
+++ b/deployment/roles/metron_streaming/files/yaf_index.template
@@ -0,0 +1,36 @@
+{
+    "template" : "yaf_index*",
+    "mappings" : {
+        "yaf_doc" : {
+            "properties": {
+                "message": {
+                    "properties": {
+                        "@timestamp":{"type":"date","format":"dateOptionalTime"},
+                        "end-time":{"type":"string"},
+                        "duration":{"type":"string"},
+                        "rtt":{"type":"string"},
+                        "proto":{"type":"string"},
+                        "sip":{"type":"string"},
+                        "sp":{"type":"string"},
+                        "dip":{"type":"string"},
+                        "dp":{"type":"string"},
+                        "iflags":{"type":"string"},
+                        "uflags":{"type":"string"},
+                        "riflags":{"type":"string"},
+                        "ruflags":{"type":"string"},
+                        "isn":{"type":"string"},
+                        "risn":{"type":"string"},
+                        "tag":{"type":"string"},
+                        "rtag":{"type":"string"},
+                        "pkt":{"type":"string"},
+                        "oct":{"type":"string"},
+                        "rpkt":{"type":"string"},
+                        "roct":{"type":"string"},
+                        "app":{"type":"string"},
+                        "end-reason":{"type":"string"}
+                    }
+                }
+            }
+        }
+    }
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index 8739529..d42e212 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -36,20 +36,36 @@
     - "etc"
     - "topologies"
 
-- name: Configure config.properties
+- name: Configure Metron topologies
   lineinfile: >
     dest={{ metron_properties_config_path }}
     regexp="{{ item.regexp }}"
     line="{{ item.line }}"
-    state=present
   with_items:
     - { regexp: "kafka.zk=", line: "kafka.zk={{ zookeeper_url }}" }
-    - { regexp: "spout.kafka.topic=", line: "spout.kafka.topic={{ pycapa_topic }}" }
     - { regexp: "es.ip=", line: "es.ip={{ groups.search[0] }}" }
     - { regexp: "es.port=", line: "es.port={{ elasticsearch_transport_port }}" }
     - { regexp: "es.clustername=", line: "es.clustername={{ elasticsearch_cluster_name }}" }
     - { regexp: "bolt.hdfs.file.system.url=", line: "bolt.hdfs.file.system.url={{ hdfs_url }}" }
+    - { regexp: "spout.kafka.topic.pcap=", line: "spout.kafka.topic.pcap={{ pycapa_topic }}" }
+    - { regexp: "spout.kafka.topic.bro=", line: "spout.kafka.topic.bro={{ bro_topic }}" }
 
-- name: Submit Pcap topology
-  command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ metron_pcap_topology_config_path }} --filter {{ metron_properties_config_path }}
+- name: Add Elasticsearch templates for topologies
+  uri:
+    url: "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}/_template/template_yaf"
+    method: POST
+    body: "{{ lookup('file','yaf_index.template') }}"
+    status_code: 200
+    body_format: json
+
+- name: Submit Metron topologies
+  command: storm jar {{ metron_directory }}/lib/{{ metron_jar_name }} org.apache.storm.flux.Flux --remote {{ item }} --filter {{ metron_properties_config_path }}
   ignore_errors: yes
+  with_items:
+    - "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/bro/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/snort/remote.yaml"
+    - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
+
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/metron_streaming/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/vars/main.yml b/deployment/roles/metron_streaming/vars/main.yml
index e3f5f38..e7004c1 100644
--- a/deployment/roles/metron_streaming/vars/main.yml
+++ b/deployment/roles/metron_streaming/vars/main.yml
@@ -5,4 +5,3 @@ metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
 metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
 metron_src_config_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs"
 metron_properties_config_path: "{{ metron_directory }}/config/etc/env/config.properties"
-metron_pcap_topology_config_path: "{{ metron_directory }}/config/topologies/pcap/remote.yaml"

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/metron_ui/files/default.json
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/files/default.json b/deployment/roles/metron_ui/files/default.json
new file mode 100644
index 0000000..f4c1d2f
--- /dev/null
+++ b/deployment/roles/metron_ui/files/default.json
@@ -0,0 +1,508 @@
+{
+  "title": "Metron",
+  "services": {
+    "query": {
+      "list": {
+        "1": {
+          "id": 1,
+          "color": "#6ED0E0",
+          "alias": "Yaf",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:yaf_doc"
+        },
+        "2": {
+          "id": 2,
+          "color": "#BA43A9",
+          "alias": "All Events",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:*_doc"
+        },
+        "3": {
+          "id": 3,
+          "color": "#BA43A9",
+          "alias": "All Alerts",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:*_alert"
+        },
+        "4": {
+          "id": 4,
+          "color": "#1F78C1",
+          "alias": "Bro Events",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:bro_doc"
+        },
+        "5": {
+          "id": 5,
+          "color": "#EF843C",
+          "alias": "Bro Alerts",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:bro_alert"
+        },
+        "6": {
+          "id": 6,
+          "color": "#BA43A9",
+          "alias": "Sourcefire Events",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:sourcefire_doc"
+        },
+        "7": {
+          "id": 7,
+          "color": "#BF1B00",
+          "alias": "Sourcefire Alerts",
+          "pin": true,
+          "type": "lucene",
+          "enable": true,
+          "query": "_type:sourcefire_alert"
+        }
+      },
+      "ids": [
+        1,
+        2,
+        3,
+        4,
+        5,
+        6,
+        7
+      ]
+    },
+    "filter": {
+      "list": {
+        "0": {
+          "type": "time",
+          "field": "timestamp",
+          "from": "now-24h",
+          "to": "now",
+          "mandate": "must",
+          "active": true,
+          "alias": "",
+          "id": 0
+        }
+      },
+      "ids": [
+        0
+      ]
+    }
+  },
+  "rows": [
+    {
+      "title": "Histogram",
+      "height": "150px",
+      "editable": true,
+      "collapse": false,
+      "collapsable": true,
+      "panels": [
+        {
+          "span": 6,
+          "editable": true,
+          "type": "histogram",
+          "loadingEditor": false,
+          "mode": "count",
+          "time_field": "timestamp",
+          "value_field": null,
+          "x-axis": true,
+          "y-axis": true,
+          "scale": 1,
+          "y_format": "short",
+          "grid": {
+            "max": null,
+            "min": 0
+          },
+          "queries": {
+            "mode": "selected",
+            "ids": [
+              4,
+              5
+            ]
+          },
+          "locked": false,
+          "annotate": {
+            "enable": false,
+            "query": "*",
+            "size": 20,
+            "field": "_type",
+            "sort": [
+              "_score",
+              "desc"
+            ]
+          },
+          "auto_int": true,
+          "resolution": 100,
+          "interval": "10m",
+          "intervals": [
+            "auto",
+            "1s",
+            "1m",
+            "5m",
+            "10m",
+            "30m",
+            "1h",
+            "3h",
+            "12h",
+            "1d",
+            "1w",
+            "1y"
+          ],
+          "lines": false,
+          "fill": 0,
+          "linewidth": 3,
+          "points": false,
+          "pointradius": 5,
+          "bars": true,
+          "stack": false,
+          "spyable": true,
+          "zoomlinks": true,
+          "options": true,
+          "legend": true,
+          "show_query": true,
+          "interactive": true,
+          "legend_counts": true,
+          "timezone": "browser",
+          "percentage": false,
+          "zerofill": true,
+          "derivative": false,
+          "tooltip": {
+            "value_type": "individual",
+            "query_as_alias": true
+          },
+          "title": "Bro Data"
+        },
+        {
+          "span": 6,
+          "editable": true,
+          "type": "histogram",
+          "loadingEditor": false,
+          "mode": "count",
+          "time_field": "timestamp",
+          "value_field": null,
+          "x-axis": true,
+          "y-axis": true,
+          "scale": 1,
+          "y_format": "none",
+          "grid": {
+            "max": null,
+            "min": 0
+          },
+          "queries": {
+            "mode": "selected",
+            "ids": [
+              6,
+              7
+            ]
+          },
+          "locked": false,
+          "annotate": {
+            "enable": false,
+            "query": "*",
+            "size": 20,
+            "field": "_type",
+            "sort": [
+              "_score",
+              "desc"
+            ]
+          },
+          "auto_int": true,
+          "resolution": 100,
+          "interval": "10m",
+          "intervals": [
+            "auto",
+            "1s",
+            "1m",
+            "5m",
+            "10m",
+            "30m",
+            "1h",
+            "3h",
+            "12h",
+            "1d",
+            "1w",
+            "1y"
+          ],
+          "lines": false,
+          "fill": 0,
+          "linewidth": 3,
+          "points": false,
+          "pointradius": 5,
+          "bars": true,
+          "stack": true,
+          "spyable": true,
+          "zoomlinks": true,
+          "options": true,
+          "legend": true,
+          "show_query": true,
+          "interactive": true,
+          "legend_counts": true,
+          "timezone": "browser",
+          "percentage": false,
+          "zerofill": true,
+          "derivative": false,
+          "tooltip": {
+            "value_type": "cumulative",
+            "query_as_alias": true
+          },
+          "title": "Sourcefire Data"
+        }
+      ],
+      "notice": false
+    },
+    {
+      "title": "Alerts",
+      "height": "150px",
+      "editable": true,
+      "collapse": false,
+      "collapsable": true,
+      "panels": [
+        {
+          "error": false,
+          "span": 12,
+          "editable": true,
+          "type": "table",
+          "loadingEditor": false,
+          "size": 10,
+          "pages": 100,
+          "offset": 0,
+          "sort": [
+            "timestamp",
+            "desc"
+          ],
+          "overflow": "min-height",
+          "fields": [
+            "_type",
+            "timestamp",
+            "priority",
+            "designated_host",
+            "description"
+          ],
+          "highlight": [],
+          "sortable": true,
+          "header": true,
+          "paging": true,
+          "field_list": false,
+          "all_fields": false,
+          "trimFactor": 400,
+          "localTime": true,
+          "timeField": "timestamp",
+          "spyable": true,
+          "queries": {
+            "mode": "selected",
+            "ids": [
+              3
+            ]
+          },
+          "locked": false,
+          "style": {
+            "font-size": "9pt"
+          },
+          "normTimes": true,
+          "title": "Alerts"
+        }
+      ],
+      "notice": false
+    },
+    {
+      "title": "Events",
+      "height": "150px",
+      "editable": true,
+      "collapse": false,
+      "collapsable": true,
+      "panels": [
+        {
+          "error": false,
+          "span": 12,
+          "editable": true,
+          "type": "table",
+          "loadingEditor": false,
+          "size": 10,
+          "pages": 100,
+          "offset": 0,
+          "sort": [
+            "message.timestamp",
+            "desc"
+          ],
+          "overflow": "min-height",
+          "fields": [
+            "_type",
+            "message.timestamp",
+            "message.original_string"
+          ],
+          "highlight": [],
+          "sortable": true,
+          "header": true,
+          "paging": true,
+          "field_list": false,
+          "all_fields": false,
+          "trimFactor": 400,
+          "localTime": true,
+          "timeField": "message.timestamp",
+          "spyable": true,
+          "queries": {
+            "mode": "selected",
+            "ids": [
+              2
+            ]
+          },
+          "locked": false,
+          "style": {
+            "font-size": "9pt"
+          },
+          "normTimes": true,
+          "title": "Events"
+        }
+      ],
+      "notice": false
+    },
+    {
+      "title": "PCAP Data",
+      "height": "150px",
+      "editable": true,
+      "collapse": false,
+      "collapsable": true,
+      "panels": [
+        {
+          "error": false,
+          "span": 12,
+          "editable": true,
+          "type": "pcap",
+          "loadingEditor": false,
+          "title": "PCAP Data"
+        },
+        {
+          "error": false,
+          "span": 12,
+          "editable": true,
+          "type": "table",
+          "loadingEditor": false,
+          "size": 100,
+          "pages": 5,
+          "offset": 0,
+          "sort": [
+            "_score",
+            "desc"
+          ],
+          "overflow": "min-height",
+          "fields": [],
+          "highlight": [],
+          "sortable": true,
+          "header": true,
+          "paging": true,
+          "field_list": true,
+          "all_fields": false,
+          "trimFactor": 300,
+          "localTime": false,
+          "timeField": "@timestamp",
+          "spyable": true,
+          "queries": {
+            "mode": "selected",
+            "ids": [
+              1
+            ]
+          },
+          "style": {
+            "font-size": "9pt"
+          },
+          "normTimes": true,
+          "title": "Yaf"
+        }
+      ],
+      "notice": false
+    }
+  ],
+  "editable": true,
+  "failover": false,
+  "index": {
+    "interval": "none",
+    "pattern": "[logstash-]YYYY.MM.DD",
+    "default": "*alert*,*_index*",
+    "warm_fields": false
+  },
+  "style": "dark",
+  "realtime": true,
+  "panel_hints": true,
+  "pulldowns": [
+    {
+      "type": "query",
+      "collapse": false,
+      "notice": false,
+      "enable": true,
+      "query": "*",
+      "pinned": true,
+      "history": [
+        "_type:sourcefire_doc",
+        "_type:bro_doc",
+        "_type:*_doc",
+        "_type:sourcefire_alert",
+        "_type:bro_alert",
+        "_type:*_alert",
+        "*"
+      ],
+      "remember": 10
+    },
+    {
+      "type": "filtering",
+      "collapse": true,
+      "notice": true,
+      "enable": true
+    }
+  ],
+  "nav": [
+    {
+      "type": "timepicker",
+      "collapse": false,
+      "notice": false,
+      "enable": true,
+      "status": "Stable",
+      "time_options": [
+        "5m",
+        "15m",
+        "1h",
+        "6h",
+        "12h",
+        "24h",
+        "2d",
+        "7d",
+        "30d"
+      ],
+      "refresh_intervals": [
+        "5s",
+        "10s",
+        "30s",
+        "1m",
+        "5m",
+        "15m",
+        "30m",
+        "1h",
+        "2h",
+        "1d"
+      ],
+      "timefield": "timestamp",
+      "now": true,
+      "filter_id": 0
+    }
+  ],
+  "loader": {
+    "save_gist": false,
+    "save_elasticsearch": true,
+    "save_local": true,
+    "save_default": true,
+    "save_temp": true,
+    "save_temp_ttl_enable": true,
+    "save_temp_ttl": "30d",
+    "load_gist": false,
+    "load_elasticsearch": true,
+    "load_elasticsearch_size": 20,
+    "load_local": false,
+    "hide": false
+  },
+  "refresh": false
+}
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/main.yml b/deployment/roles/metron_ui/tasks/main.yml
index 635bc66..69946b6 100644
--- a/deployment/roles/metron_ui/tasks/main.yml
+++ b/deployment/roles/metron_ui/tasks/main.yml
@@ -43,5 +43,11 @@
     replace: "pcap_*"
     backup: yes
 
+- name: Configure default dashboard
+  copy:
+    src: default.json
+    dest: /root/.nvm/v0.10.35/lib/node_modules/opensoc-ui/lib/static_dist/app/dashboards/
+    mode: 0644
+
 - name: Start Metron UI service
   shell: source ~/.bashrc && opensoc-ui start
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/snort/tasks/flume.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/flume.yml b/deployment/roles/snort/tasks/flume.yml
index df93b52..5bb5b0b 100644
--- a/deployment/roles/snort/tasks/flume.yml
+++ b/deployment/roles/snort/tasks/flume.yml
@@ -4,24 +4,12 @@
   with_items:
     - { src: flume-snort.conf, dest: /etc/flume/conf/flume-snort.conf }
 
-- name: Set kafka broker in flume
-  lineinfile: >
-    dest="{{ item.dest }}"
-    regexp="{{ item.regexp }}"
-    line="{{ item.line }}"
-    state=present
+- name: Configure destination for snort alerts
+  lineinfile: dest=/etc/flume/conf/flume-snort.conf regexp={{ item.regexp }} line={{ item.line }}
   with_items:
-    - { dest: /etc/flume/conf/flume-snort.conf,
-        regexp: '^snort\.sinks\.kafka-sink\.brokerList.*$',
+    - { regexp: '^snort\.sinks\.kafka-sink\.brokerList.*$',
         line: 'snort.sinks.kafka-sink.brokerList = {{ kafka_broker_url }}' }
-
-- name: Set kafka topics in flume
-  lineinfile: >
-    dest="{{ item.dest }}"
-    regexp="{{ item.regexp }}"
-    line="{{ item.line }}"
-    state=present
-  with_items:
-    - { dest: /etc/flume/conf/flume-snort.conf,
-        regexp: '^snort\.sinks\.kafka-sink\.topic.*$',
+    - { regexp: '^snort\.sinks\.kafka-sink\.topic.*$',
         line: 'snort.sinks.kafka-sink.topic = {{ snort_topic }}'}
+    - { regexp: '^snort.sources.exec-source.command.*$',
+        line: 'snort.sources.exec-source.command = tail -F {{ snort_alert_csv_path }}' }

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/snort/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/main.yml b/deployment/roles/snort/tasks/main.yml
index 651a8a8..7e7029a 100644
--- a/deployment/roles/snort/tasks/main.yml
+++ b/deployment/roles/snort/tasks/main.yml
@@ -1,4 +1,7 @@
 ---
+- name: Install libselinux-python
+  yum: name=libselinux-python
+
 - include: snort.yml
 
 - include: flume.yml

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/snort/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/snort.yml b/deployment/roles/snort/tasks/snort.yml
index 69652e0..66e770e 100644
--- a/deployment/roles/snort/tasks/snort.yml
+++ b/deployment/roles/snort/tasks/snort.yml
@@ -1,4 +1,7 @@
 ---
+- name: Install EPEL repository
+  yum: name=epel-release
+
 - name: Install development tools
   yum: name="@Development tools" state=present
 
@@ -67,11 +70,20 @@
 - name: Download snort configuration
   copy: src=snort.conf dest=/etc/snort/snort.conf
 
-- name: Configure snort
-  lineinfile: >
-    dest=/etc/snort/snort.conf
-    regexp="{{ item.regexp }}"
-    line="{{ item.line }}"
-    state=present
+- name: Configure network
+  lineinfile: dest=/etc/snort/snort.conf regexp="{{ item.regexp }}" line="{{ item.line }}"
   with_items:
     - { regexp: "^ipvar HOME_NET.*$", line: "ipvar HOME_NET {{ ansible_eth0.ipv4.address }}" }
+
+- name: Configure alerting
+  lineinfile:
+    dest: /etc/snort/snort.conf
+    line: "output alert_csv: {{ snort_alert_csv_path }} default"
+
+- name: Configure sysconfig
+  lineinfile: dest=/etc/sysconfig/snort regexp="{{ item.regexp }}" line="{{ item.line }}"
+  with_items:
+    - { regexp: "^ALERTMODE=.*$", line: "ALERTMODE=" }
+    - { regexp: "^NO_PACKET_LOG=.*$", line: "NO_PACKET_LOG=1" }
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/yaf/files/start-yaf.sh
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/files/start-yaf.sh b/deployment/roles/yaf/files/start-yaf.sh
new file mode 100644
index 0000000..af52486
--- /dev/null
+++ b/deployment/roles/yaf/files/start-yaf.sh
@@ -0,0 +1 @@
+/usr/local/bin/yaf --in eth0 --live pcap --idle-timeout 0 | /usr/local/bin/yafscii --tabular --print-header | /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh --broker-list $1 --topic ipfix
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
index 64eb08c..552fb44 100644
--- a/deployment/roles/yaf/tasks/main.yml
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -1,10 +1,74 @@
 ---
-- include: yaf.yml
+- name: Install development tools
+  yum: name="@Development tools" state=present
 
-- include: flume.yml
+- name: Install prerequisites
+  yum: name={{ item }}
+  with_items:
+    - libpcap
+    - libpcap-devel
+    - pcre
+    - pcre-devel
+    - glib2-devel
+    - zlib
+    - zlib-devel
 
-- name: Start flume service to consume ipfix
-  service: name=flume-agent state=restarted args=ipfix
+- name: Download fixbuf
+  get_url:
+    url: "http://tools.netsa.cert.org/releases/libfixbuf-{{fixbuf_version}}.tar.gz"
+    dest: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
 
-- name: Start yaf service to produce ipfix
-  service: name=yaf state=restarted
+- name: Extract fixbuf tarball
+  unarchive:
+    src: "/tmp/libfixbuf-{{fixbuf_version}}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: "/tmp/libfixbuf-{{fixbuf_version}}"
+
+- name: Compile and Install fixbuf
+  shell: "{{item}}"
+  args:
+    chdir: "/tmp/libfixbuf-{{fixbuf_version}}"
+  with_items:
+    - ./configure
+    - make
+    - make install
+
+- name: Download yaf
+  get_url:
+    url: "http://tools.netsa.cert.org/releases/yaf-{{yaf_version}}.tar.gz"
+    dest: "/tmp/yaf-{{yaf_version}}.tar.gz"
+
+- name: Extract yaf tarball
+  unarchive:
+    src: "/tmp/yaf-{{yaf_version}}.tar.gz"
+    dest: /tmp
+    copy: no
+    creates: /usr/local/bin/yaf
+
+- name: Compile and install yaf
+  shell: "{{item}}"
+  args:
+    chdir: "/tmp/yaf-{{yaf_version}}"
+    creates: /usr/local/bin/yaf
+  with_items:
+    - ./configure --enable-applabel --enable-plugins
+    - make
+    - make install
+
+- name: Create Yaf home directory
+  file:
+    path: "{{ yaf_home }}"
+    state: directory
+    mode: 0755
+
+- name: Add Yaf startup script
+  copy:
+    src: start-yaf.sh
+    dest: "{{ yaf_home }}/start-yaf.sh"
+    mode: 0755
+
+- name: Start yaf
+  shell: "daemonize -c {{ yaf_home }} -e /var/log/yaf.log -o /var/log/yaf.log -l /var/lock/subsys/yaf {{yaf_home}}/start-yaf.sh {{ kafka_broker_url }}"
+  args:
+    creates: /var/lock/subsys/yaf
\ No newline at end of file

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/yaf/tasks/yaf.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/yaf.yml b/deployment/roles/yaf/tasks/yaf.yml
index 351d174..a65830e 100644
--- a/deployment/roles/yaf/tasks/yaf.yml
+++ b/deployment/roles/yaf/tasks/yaf.yml
@@ -56,22 +56,19 @@
     - make
     - make install
 
-- name: Create yaf service
-  shell: "{{item}}"
-  args:
-    creates: /usr/local/etc/yaf.conf
-  with_items:
-    - cp /tmp/yaf-{{yaf_version}}/etc/init.d/yaf /etc/init.d/yaf
-    - chmod +x /etc/init.d/yaf
-    - cp /tmp/yaf-{{yaf_version}}/etc/yaf.conf /usr/local/etc/yaf.conf
+- name: Create Yaf home directory
+  file:
+    path: "{{ yaf_home }}"
+    state: directory
+    mode: 0755
 
-- name: Configure yaf
-  lineinfile: >
-    dest=/usr/local/etc/yaf.conf
-    regexp="{{ item.regexp }}"
-    line="{{ item.line }}"
-    state=present
-  with_items:
-    - { regexp: '^ENABLED=.*$', line: 'ENABLED=1' }
-    - { regexp: '^YAF_EXTRAFLAGS=.*$', line: 'YAF_EXTRAFLAGS='}
-    - { regexp: '^YAF_CAP_IF=.*$', line: 'YAF_CAP_IF={{ iface }}'}
+- name: Add Yaf startup script
+  copy:
+    src: start-yaf.sh
+    dest: "{{ yaf_home }}/start-yaf.sh"
+    mode: 0755
+
+- name: Start yaf
+  shell: "daemonize -c {{ yaf_home }} -e /var/log/yaf.log -o /var/log/yaf.log -l /var/lock/subsys/yaf {{yaf_home}}/start-yaf.sh {{ kafka_broker_url }}"
+  args:
+    creates: /var/lock/subsys/yaf

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/deployment/roles/yaf/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/vars/main.yml b/deployment/roles/yaf/vars/main.yml
index b73b8d1..d88794b 100644
--- a/deployment/roles/yaf/vars/main.yml
+++ b/deployment/roles/yaf/vars/main.yml
@@ -1,3 +1,4 @@
 ---
 fixbuf_version: 1.7.1
 yaf_version: 2.8.0
+yaf_home: /opt/yaf

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicSnortParser.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicSnortParser.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicSnortParser.java
new file mode 100644
index 0000000..91fe312
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicSnortParser.java
@@ -0,0 +1,134 @@
+package org.apache.metron.parsing.parsers;
+
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+
+import org.apache.metron.parser.interfaces.MessageParser;
+import org.json.simple.JSONObject;
+
+import java.text.ParseException;
+
+@SuppressWarnings("serial")
+public class BasicSnortParser extends AbstractParser implements MessageParser {
+
+	/**
+	 * The default field names for Snort Alerts.
+	 */
+	private String[] fieldNames = new String[] {
+			"timestamp",
+			"sig_generator",
+			"sig_id",
+			"sig_rev",
+			"msg",
+			"proto",
+			"src",
+			"srcport",
+			"dst",
+			"dstport",
+			"ethsrc",
+			"ethdst",
+			"ethlen",
+			"tcpflags",
+			"tcpseq",
+			"tcpack",
+			"tcplen",
+			"tcpwindow",
+			"ttl",
+			"tos",
+			"id",
+			"dgmlen",
+			"iplen",
+			"icmptype",
+			"icmpcode",
+			"icmpid",
+			"icmpseq"
+	};
+
+	/**
+	 * Snort alerts are received as CSV records
+	 */
+	private String recordDelimiter = ",";
+
+	@Override
+	public JSONObject parse(byte[] rawMessage) {
+
+		JSONObject jsonMessage = new JSONObject();
+		try {
+			// snort alerts expected as csv records
+			String csvMessage = new String(rawMessage, "UTF-8");
+			String[] records = csvMessage.split(recordDelimiter, -1);
+
+			// validate the number of fields
+			if (records.length != fieldNames.length) {
+				throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.length);
+			}
+
+			// build the json record from each field
+			for (int i=0; i<records.length; i++) {
+			
+				String field = fieldNames[i];
+				String record = records[i];
+				
+				if("timestamp".equals(field)) {
+
+					// convert the timestamp to epoch
+					jsonMessage.put("timestamp", toEpoch(record));
+					
+				} else {
+					jsonMessage.put(field, record);
+				}
+			}
+
+			// add original msg; required by 'checkForSchemaCorrectness'
+			jsonMessage.put("original_string", csvMessage);
+
+		} catch (Exception e) {
+
+            _LOG.error("unable to parse message: " + rawMessage);
+            e.printStackTrace();
+            return null;
+        }
+
+		return jsonMessage;
+	}
+
+	/**
+	 * Parses Snort's default date-time representation and
+	 * converts to epoch.
+	 * @param datetime Snort's default date-time as String '01/27-16:01:04.877970'
+	 * @return epoch time
+	 * @throws java.text.ParseException 
+	 */
+	private long toEpoch(String snortDatetime) throws ParseException {
+		
+		/*
+		 * TODO how does Snort not embed the year in their default timestamp?! need to change this in 
+		 * Snort configuration.  for now, just assume current year.
+		 */
+		int year = Calendar.getInstance().get(Calendar.YEAR);
+		String withYear = Integer.toString(year) + " " + snortDatetime;
+		
+		// convert to epoch time
+		SimpleDateFormat df = new SimpleDateFormat("yyyy MM/dd-HH:mm:ss.S");
+		Date date = df.parse(withYear);
+		return date.getTime();
+	}
+
+	public String getRecordDelimiter() {
+		return this.recordDelimiter;
+	}
+
+	public void setRecordDelimiter(String recordDelimiter) {
+		this.recordDelimiter = recordDelimiter;
+	}
+
+	public String[] getFieldNames() {
+		return this.fieldNames;
+	}
+
+	public void setFieldNames(String[] fieldNames) {
+		this.fieldNames = fieldNames;
+	}
+
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicYafParser.java
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicYafParser.java b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicYafParser.java
new file mode 100644
index 0000000..48823e4
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/java/org/apache/metron/parsing/parsers/BasicYafParser.java
@@ -0,0 +1,191 @@
+package org.apache.metron.parsing.parsers;
+
+import oi.thekraken.grok.api.Grok;
+import oi.thekraken.grok.api.Match;
+import oi.thekraken.grok.api.exception.GrokException;
+import org.apache.commons.io.IOUtils;
+import org.json.simple.JSONObject;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.Map;
+
+/**
+ * Created by rmerriman on 1/27/16.
+ */
+public class BasicYafParser extends AbstractParser {
+
+
+  /**
+   * The default field names for Snort Alerts.
+   */
+  private String[] fieldNames = new String[] {
+          "start-time",
+          "end-time",
+          "duration",
+          "rtt",
+          "proto",
+          "sip",
+          "sp",
+          "dip",
+          "dp",
+          "iflags",
+          "uflags",
+          "riflags",
+          "ruflags",
+          "isn",
+          "risn",
+          "tag",
+          "rtag",
+          "pkt",
+          "oct",
+          "rpkt",
+          "roct",
+          "app",
+          "end-reason"
+  };
+
+  private String recordDelimiter = "\\|";
+
+  private transient  Grok  grok;
+  private transient InputStream pattern_url;
+
+  public static final String PREFIX = "stream2file";
+  public static final String SUFFIX = ".tmp";
+
+  public static File stream2file(InputStream in) throws IOException {
+    final File tempFile = File.createTempFile(PREFIX, SUFFIX);
+    tempFile.deleteOnExit();
+    try (FileOutputStream out = new FileOutputStream(tempFile)) {
+      IOUtils.copy(in, out);
+    }
+    return tempFile;
+  }
+
+  protected static final Logger _LOG = LoggerFactory
+          .getLogger(BasicBroParser.class);
+  private JSONCleaner cleaner = new JSONCleaner();
+
+  @Override
+  public void init() {
+    // pattern_url = Resources.getResource("patterns/asa");
+
+    pattern_url = getClass().getClassLoader().getResourceAsStream(
+            "patterns/yaf");
+
+    File file = null;
+    try {
+      file = stream2file(pattern_url);
+    } catch (IOException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+    try {
+      grok = Grok.create(file.getPath());
+    } catch (GrokException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+
+    try {
+      grok.compile("%{YAF_DELIMITED}");
+    } catch (GrokException e) {
+      // TODO Auto-generated catch block
+      e.printStackTrace();
+    }
+  }
+
+  @SuppressWarnings("unchecked")
+  public JSONObject parse(byte[] msg) {
+    //return parseManual(msg);
+    return parseWithGrok(msg);
+  }
+
+  private JSONObject parseWithGrok(byte[] msg) {
+    _LOG.trace("[Metron] Starting to parse incoming message with grok");
+    JSONObject jsonMessage = new JSONObject();
+    try {
+      String rawMessage = new String(msg, "UTF-8");
+      System.out.println("Received message: " + rawMessage);
+
+      Match gm = grok.match(rawMessage);
+      gm.captures();
+      Map grokMap = gm.toMap();
+      jsonMessage.putAll(gm.toMap());
+
+      jsonMessage.put("original_string", rawMessage);
+      String startTime = (String) grokMap.get("start_time");
+      if (startTime != null) {
+        jsonMessage.put("timestamp", toEpoch(startTime));
+      } else {
+        jsonMessage.put("timestamp", "0");
+      }
+      String endTime = (String) grokMap.get("end_time");
+      if (endTime != null) {
+        jsonMessage.put("end_time", toEpoch(endTime));
+      } else {
+        jsonMessage.put("end_time", "0");
+      }
+      jsonMessage.remove("YAF_DELIMITED");
+      jsonMessage.remove("start_time");
+    } catch (Exception e) {
+      e.printStackTrace();
+      return null;
+    }
+    return jsonMessage;
+  }
+
+  private JSONObject parseManual(byte[] msg) {
+    _LOG.trace("[Metron] Starting to parse incoming message");
+
+    JSONObject jsonMessage = new JSONObject();
+    try {
+      String rawMessage = new String(msg, "UTF-8");
+      String[] records = rawMessage.split(recordDelimiter, -1);
+
+      // validate the number of fields
+      if (records.length != fieldNames.length) {
+        throw new IllegalArgumentException("Unexpected number of fields, expected: " + fieldNames.length + " got: " + records.length);
+      }
+
+      // build the json record from each field
+      for (int i=0; i<records.length; i++) {
+
+        String field = fieldNames[i];
+        String record = records[i].trim();
+
+        if("start-time".equals(field) || "end-time".equals(field)) {
+          // convert the timestamp to epoch
+          String timeFieldName = "start-time".equals(field) ? "timestamp" : field;
+          jsonMessage.put(timeFieldName, toEpoch(record));
+        } else {
+          jsonMessage.put(field, record);
+        }
+      }
+
+      jsonMessage.put("original_string", rawMessage);
+
+    } catch (Exception e) {
+
+      _LOG.error("Unable to Parse Message: " + msg);
+      e.printStackTrace();
+      return null;
+    }
+
+    return jsonMessage;
+  }
+
+  private long toEpoch(String datetime) throws ParseException {
+    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.S z");
+    Date date = sdf.parse(datetime + " UTC");
+    return date.getTime();
+  }
+}

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf b/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
new file mode 100644
index 0000000..3ac640e
--- /dev/null
+++ b/metron-streaming/Metron-MessageParsers/src/main/resources/patterns/yaf
@@ -0,0 +1,113 @@
+# Forked from https://github.com/elasticsearch/logstash/tree/v1.4.0/patterns
+
+USERNAME [a-zA-Z0-9._-]+
+USER %{USERNAME:UNWANTED}
+INT (?:[+-]?(?:[0-9]+))
+BASE10NUM (?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))
+NUMBER (?:%{BASE10NUM:UNWANTED})
+BASE16NUM (?<![0-9A-Fa-f])(?:[+-]?(?:0x)?(?:[0-9A-Fa-f]+))
+BASE16FLOAT \b(?<![0-9A-Fa-f.])(?:[+-]?(?:0x)?(?:(?:[0-9A-Fa-f]+(?:\.[0-9A-Fa-f]*)?)|(?:\.[0-9A-Fa-f]+)))\b
+
+POSINT \b(?:[1-9][0-9]*)\b
+NONNEGINT \b(?:[0-9]+)\b
+WORD \b\w+\b
+NOTSPACE \S+
+SPACE \s*
+DATA .*?
+GREEDYDATA .*
+#QUOTEDSTRING (?:(?<!\\)(?:"(?:\\.|[^\\"])*"|(?:'(?:\\.|[^\\'])*')|(?:`(?:\\.|[^\\`])*`)))
+QUOTEDSTRING (?>(?<!\\)(?>"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``))
+UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}
+
+# Networking
+MAC (?:%{CISCOMAC:UNWANTED}|%{WINDOWSMAC:UNWANTED}|%{COMMONMAC:UNWANTED})
+CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})
+WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})
+COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})
+IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5
 ]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?
+IPV4 (?<![0-9])(?:(?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2})[.](?:25[0-5]|2[0-4][0-9]|[0-1]?[0-9]{1,2}))(?![0-9])
+IP (?:%{IPV6:UNWANTED}|%{IPV4:UNWANTED})
+HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)
+HOST %{HOSTNAME:UNWANTED}
+IPORHOST (?:%{HOSTNAME:UNWANTED}|%{IP:UNWANTED})
+HOSTPORT (?:%{IPORHOST}:%{POSINT:PORT})
+
+# paths
+PATH (?:%{UNIXPATH}|%{WINPATH})
+UNIXPATH (?>/(?>[\w_%!$@:.,~-]+|\\.)*)+
+#UNIXPATH (?<![\w\/])(?:/[^\/\s?*]*)+
+TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))
+WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+
+URIPROTO [A-Za-z]+(\+[A-Za-z+]+)?
+URIHOST %{IPORHOST}(?::%{POSINT:port})?
+# uripath comes loosely from RFC1738, but mostly from what Firefox
+# doesn't turn into %XX
+URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+
+#URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)?
+URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
+URIPATHPARAM %{URIPATH}(?:%{URIPARAM})?
+URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?
+
+# Months: January, Feb, 3, 03, 12, December
+MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b
+MONTHNUM (?:0?[1-9]|1[0-2])
+MONTHNUM2 (?:0[1-9]|1[0-2])
+MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])
+
+# Days: Monday, Tue, Thu, etc...
+DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)
+
+# Years?
+YEAR (?>\d\d){1,2}
+# Time: HH:MM:SS
+#TIME \d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?
+# I'm still on the fence about using grok to perform the time match,
+# since it's probably slower.
+# TIME %{POSINT<24}:%{POSINT<60}(?::%{POSINT<60}(?:\.%{POSINT})?)?
+HOUR (?:2[0123]|[01]?[0-9])
+MINUTE (?:[0-5][0-9])
+# '60' is a leap second in most time standards and thus is valid.
+SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)
+TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9])
+# datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it)
+DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}
+DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}
+ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE}))
+ISO8601_SECOND (?:%{SECOND}|60)
+TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?
+DATE %{DATE_US}|%{DATE_EU}
+DATESTAMP %{DATE}[- ]%{TIME}
+TZ (?:[PMCE][SD]T|UTC)
+DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}
+DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}
+DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}
+DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}
+GREEDYDATA .*
+
+# Syslog Dates: Month Day HH:MM:SS
+SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME}
+PROG (?:[\w._/%-]+)
+SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])?
+SYSLOGHOST %{IPORHOST}
+SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}>
+HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}
+
+# Shortcuts
+QS %{QUOTEDSTRING:UNWANTED}
+
+# Log formats
+SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:
+
+MESSAGESLOG %{SYSLOGBASE} %{DATA}
+
+COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)
+COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}
+
+# Log Levels
+LOGLEVEL ([A|a]lert|ALERT|[T|t]race|TRACE|[D|d]ebug|DEBUG|[N|n]otice|NOTICE|[I|i]nfo|INFO|[W|w]arn?(?:ing)?|WARN?(?:ING)?|[E|e]rr?(?:or)?|ERR?(?:OR)?|[C|c]rit?(?:ical)?|CRIT?(?:ICAL)?|[F|f]atal|FATAL|[S|s]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)
+
+# Yaf
+YAF_TIME_FORMAT %{YEAR:UNWANTED}-%{MONTHNUM:UNWANTED}-%{MONTHDAY:UNWANTED}[T ]%{HOUR:UNWANTED}:%{MINUTE:UNWANTED}:%{SECOND:UNWANTED}
+YAF_DELIMITED %{YAF_TIME_FORMAT:start_time}\|%{YAF_TIME_FORMAT:end_time}\|%{SPACE:UNWANTED}%{BASE10NUM:duration}\|%{SPACE:UNWANTED}%{BASE10NUM:rtt}\|%{SPACE:UNWANTED}%{INT:proto}\|%{SPACE:UNWANTED}%{IP:sip}\|%{SPACE:UNWANTED}%{INT:sp}\|%{SPACE:UNWANTED}%{IP:dip}\|%{SPACE:UNWANTED}%{INT:dp}\|%{SPACE:UNWANTED}%{DATA:iflags}\|%{SPACE:UNWANTED}%{DATA:uflags}\|%{SPACE:UNWANTED}%{DATA:riflags}\|%{SPACE:UNWANTED}%{DATA:ruflags}\|%{SPACE:UNWANTED}%{WORD:isn}\|%{SPACE:UNWANTED}%{DATA:risn}\|%{SPACE:UNWANTED}%{DATA:tag}\|%{GREEDYDATA:rtag}\|%{SPACE:UNWANTED}%{INT:pkt}\|%{SPACE:UNWANTED}%{INT:oct}\|%{SPACE:UNWANTED}%{INT:rpkt}\|%{SPACE:UNWANTED}%{INT:roct}\|%{SPACE:UNWANTED}%{INT:app}\|%{GREEDYDATA:end_reason}
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
index 4c900d4..3c8c1a0 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/etc/env/config.properties
@@ -1,7 +1,15 @@
 ##### Kafka #####
 
 kafka.zk=zkpr1:2181,zkpr2:2181,zkpr3:2181
-spout.kafka.topic=bro_raw
+spout.kafka.topic.asa=asa
+spout.kafka.topic.bro=bro_raw
+spout.kafka.topic.fireeye=fireeye
+spout.kafka.topic.ise=ise
+spout.kafka.topic.lancope=lancope
+spout.kafka.topic.paloalto=paloalto
+spout.kafka.topic.pcap=pcap
+spout.kafka.topic.snort=snort
+spout.kafka.topic.yaf=ipfix
 
 ##### ElasticSearch #####
 

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
index 22c3d32..e3b01d6 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/asa/remote.yaml
@@ -130,11 +130,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.asa}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.asa}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
index d3e6326..956e346 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/bro/remote.yaml
@@ -78,11 +78,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.bro}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.bro}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
index 487333d..b759adf 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/fireeye/remote.yaml
@@ -130,11 +130,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.fireeye}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.fireeye}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
index 6180f02..d425269 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/ise/remote.yaml
@@ -78,11 +78,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.ise}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.ise}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
index 75a3180..7ccc065 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/lancope/remote.yaml
@@ -119,11 +119,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.lancope}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.lancope}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
index 2c0b928..a852dd1 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/paloalto/remote.yaml
@@ -110,11 +110,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.paloalto}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.paloalto}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
index e3a130c..3ea364d 100644
--- a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/pcap/remote.yaml
@@ -74,11 +74,11 @@ components:
             # zookeeper hosts
             - ref: "zkHosts"
             # topic name
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.pcap}"
             # zk root
             - ""
             # id
-            - "${spout.kafka.topic}"
+            - "${spout.kafka.topic.pcap}"
         properties:
             -   name: "forceFromStart"
                 value: true

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml
new file mode 100644
index 0000000..498c747
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/local.yaml
@@ -0,0 +1,188 @@
+name: "snort-local"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "snortParser"
+        className: "org.apache.metron.parsing.parsers.BasicSnortParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+
+
+spouts:
+    -   id: "testingSpout"
+        className: "org.apache.metron.test.spouts.GenericInternalTestSpout"
+        parallelism: 1
+        configMethods:
+            -   name: "withFilename"
+                args:
+                    - "SampleInput/SourcefireExampleOutput"
+            -   name: "withRepeating"
+                args:
+                    - true
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "snortParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "snort"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "snort_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "snort_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "snort_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+
+

http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/54c8de38/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
new file mode 100644
index 0000000..e813149
--- /dev/null
+++ b/metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs/topologies/snort/remote.yaml
@@ -0,0 +1,201 @@
+name: "snort"
+config:
+    topology.workers: 1
+
+components:
+    -   id: "snortParser"
+        className: "org.apache.metron.parsing.parsers.BasicSnortParser"
+    -   id: "genericMessageFilter"
+        className: "org.apache.metron.filters.GenericMessageFilter"
+    -   id: "indexAdapter"
+        className: "org.apache.metron.indexing.adapters.ESTimedRotatingAdapter"
+    -   id: "metricConfig"
+        className: "org.apache.commons.configuration.BaseConfiguration"
+        configMethods:
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.graphite"
+                    - "${org.apache.metron.metrics.reporter.graphite}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.console"
+                    - "${org.apache.metron.metrics.reporter.console}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.reporter.jmx"
+                    - "${org.apache.metron.metrics.reporter.jmx}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.address"
+                    - "${org.apache.metron.metrics.graphite.address}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.graphite.port"
+                    - "${org.apache.metron.metrics.graphite.port}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryParserBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryParserBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.acks"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.emits"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.GenericEnrichmentBolt.fails"
+                    - "${org.apache.metron.metrics.GenericEnrichmentBolt.fails}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.acks"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.acks}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.emits"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.emits}"
+            -   name: "setProperty"
+                args:
+                    - "org.apache.metron.metrics.TelemetryIndexingBolt.fails"
+                    - "${org.apache.metron.metrics.TelemetryIndexingBolt.fails}"
+    -   id: "zkHosts"
+        className: "storm.kafka.ZkHosts"
+        constructorArgs:
+            - "${kafka.zk}"
+    -   id: "kafkaConfig"
+        className: "storm.kafka.SpoutConfig"
+        constructorArgs:
+            # zookeeper hosts
+            - ref: "zkHosts"
+            # topic name
+            - "${spout.kafka.topic.snort}"
+            # zk root
+            - ""
+            # id
+            - "${spout.kafka.topic.snort}"
+        properties:
+            -   name: "forceFromStart"
+                value: true
+            -   name: "startOffsetTime"
+                value: -1
+
+spouts:
+    -   id: "kafkaSpout"
+        className: "storm.kafka.KafkaSpout"
+        constructorArgs:
+            - ref: "kafkaConfig"
+
+bolts:
+    -   id: "parserBolt"
+        className: "org.apache.metron.parsing.TelemetryParserBolt"
+        configMethods:
+            -   name: "withMessageParser"
+                args:
+                    - ref: "snortParser"
+            -   name: "withOutputFieldName"
+                args:
+                    - "snort"
+            -   name: "withMessageFilter"
+                args:
+                    - ref: "genericMessageFilter"
+            -   name: "withMetricConfig"
+                args:
+                    - ref: "metricConfig"
+    -   id: "indexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "snort_index"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM.dd.hh"
+            -   name: "withDocumentName"
+                args:
+                    - "snort_doc"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+    -   id: "errorIndexingBolt"
+        className: "org.apache.metron.indexing.TelemetryIndexingBolt"
+        configMethods:
+            -   name: "withIndexIP"
+                args:
+                    - "${es.ip}"
+            -   name: "withIndexPort"
+                args:
+                    - ${es.port}
+            -   name: "withClusterName"
+                args:
+                    - "${es.clustername}"
+            -   name: "withIndexName"
+                args:
+                    - "error"
+            -   name: "withIndexTimestamp"
+                args:
+                    - "yyyy.MM"
+            -   name: "withDocumentName"
+                args:
+                    - "snort_error"
+            -   name: "withBulk"
+                args:
+                    - 1
+            -   name: "withIndexAdapter"
+                args:
+                    - ref: "indexAdapter"
+            -   name: "withMetricConfiguration"
+                args:
+                    - ref: "metricConfig"
+
+streams:
+    -   name: "spout -> parser"
+        from: "kafkaSpout"
+        to: "parserBolt"
+        grouping:
+            type: SHUFFLE
+    -   name: "parser -> indexing"
+        from: "parserBolt"
+        to: "indexingBolt"
+        grouping:
+            streamId: "message"
+            type: FIELDS
+            args: ["key"]
+    -   name: "parser -> errors"
+        from: "parserBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+    -   name: "indexing -> errors"
+        from: "indexingBolt"
+        to: "errorIndexingBolt"
+        grouping:
+            streamId: "error"
+            type: SHUFFLE
+
+