You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/21 18:06:25 UTC
[43/43] incubator-metron git commit: METRON-77: Refactor Deployment
Scripts to Accept Changes from METRON-72 (dlyle65535 via cestella) closes
apache/incubator-metron#53
METRON-77: Refactor Deployment Scripts to Accept Changes from METRON-72 (dlyle65535 via cestella) closes apache/incubator-metron#53
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/63432776
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/63432776
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/63432776
Branch: refs/heads/Metron_0.1BETA
Commit: 634327763bb9807bdce9d63c99979272f95474a4
Parents: a44cc01
Author: dlyle65535 <dl...@gmail.com>
Authored: Mon Mar 21 13:01:33 2016 -0400
Committer: cstella <ce...@gmail.com>
Committed: Mon Mar 21 13:01:33 2016 -0400
----------------------------------------------------------------------
.gitignore | 1 +
deployment/amazon-ec2/playbook.yml | 1 +
.../inventory/singlenode-vagrant/group_vars/all | 6 +-
deployment/roles/bro/meta/main.yml | 2 +
.../elasticsearch/files/yaf_index.template | 114 +++++++++++++------
deployment/roles/kafka-client/tasks/main.yml | 30 +++++
.../roles/metron_streaming/defaults/main.yml | 2 +-
deployment/roles/metron_ui/defaults/main.yml | 21 ++++
.../roles/metron_ui/tasks/copy-source.yml | 46 ++++++++
deployment/roles/metron_ui/tasks/main.yml | 24 ++--
deployment/roles/metron_ui/vars/main.yml | 22 ----
deployment/roles/pycapa/meta/main.yml | 1 +
deployment/roles/snort/meta/main.yml | 2 +
deployment/roles/yaf/meta/main.yml | 1 +
deployment/roles/yaf/tasks/kafka-client.yml | 30 -----
deployment/roles/yaf/tasks/main.yml | 1 -
.../vagrant/singlenode-vagrant/ansible.cfg | 1 +
metron-streaming/Metron-DataLoads/pom.xml | 2 -
18 files changed, 199 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 488d7ea..6434855 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,3 +18,4 @@ target
*.settings
*hbase-site.xml
log4j.properties
+*.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/playbook.yml b/deployment/amazon-ec2/playbook.yml
index 5914b98..2b2d060 100644
--- a/deployment/amazon-ec2/playbook.yml
+++ b/deployment/amazon-ec2/playbook.yml
@@ -47,6 +47,7 @@
- include: tasks/check-hosts.yml
tags:
- ec2
+ - wait
#
# mount additional data volumes on all ec2 hosts
#
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index a5e0af7..c1c199b 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -60,17 +60,17 @@ ambari_server_mem: 512
threat_intel_bulk_load: False
#Sensors
-install_pycapa: True
+install_pycapa: False
install_bro: True
install_snort: True
-install_yaf: False
+install_yaf: True
pcap_replay: True
sniff_interface: eth1
pcap_replay_interface: "{{ sniff_interface }}"
storm_topologies:
- "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- - "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
- "{{ metron_directory }}/config/topologies/snort/remote.yaml"
+ - "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
pcapservice_port: 8081
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/bro/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/bro/meta/main.yml b/deployment/roles/bro/meta/main.yml
index 4d10740..320ce8d 100644
--- a/deployment/roles/bro/meta/main.yml
+++ b/deployment/roles/bro/meta/main.yml
@@ -19,3 +19,5 @@ dependencies:
- ambari_gather_facts
- libselinux-python
- build-tools
+ - kafka-client
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/elasticsearch/files/yaf_index.template
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/files/yaf_index.template b/deployment/roles/elasticsearch/files/yaf_index.template
index c8c1702..15ff6f8 100644
--- a/deployment/roles/elasticsearch/files/yaf_index.template
+++ b/deployment/roles/elasticsearch/files/yaf_index.template
@@ -1,36 +1,82 @@
{
- "template" : "yaf_index*",
- "mappings" : {
- "yaf_doc" : {
- "properties": {
- "message": {
- "properties": {
- "@timestamp":{"type":"date","format":"dateOptionalTime"},
- "end-time":{"type":"string"},
- "duration":{"type":"string"},
- "rtt":{"type":"string"},
- "proto":{"type":"string"},
- "sip":{"type":"string"},
- "sp":{"type":"string"},
- "dip":{"type":"string"},
- "dp":{"type":"string"},
- "iflags":{"type":"string"},
- "uflags":{"type":"string"},
- "riflags":{"type":"string"},
- "ruflags":{"type":"string"},
- "isn":{"type":"string"},
- "risn":{"type":"string"},
- "tag":{"type":"string"},
- "rtag":{"type":"string"},
- "pkt":{"type":"string"},
- "oct":{"type":"string"},
- "rpkt":{"type":"string"},
- "roct":{"type":"string"},
- "app":{"type":"string"},
- "end-reason":{"type":"string"}
- }
- }
+ "template": "yaf_index*",
+ "mappings": {
+ "yaf_doc": {
+ "properties": {
+ "timestamp": {
+ "type": "date",
+ "format": "dateOptionalTime"
+ },
+ "location_point": {
+ "type": "geo_point"
+ },
+ "end-time": {
+ "type": "string"
+ },
+ "duration": {
+ "type": "string"
+ },
+ "rtt": {
+ "type": "string"
+ },
+ "proto": {
+ "type": "string"
+ },
+ "sip": {
+ "type": "string"
+ },
+ "sp": {
+ "type": "string"
+ },
+ "dip": {
+ "type": "string"
+ },
+ "dp": {
+ "type": "string"
+ },
+ "iflags": {
+ "type": "string"
+ },
+ "uflags": {
+ "type": "string"
+ },
+ "riflags": {
+ "type": "string"
+ },
+ "ruflags": {
+ "type": "string"
+ },
+ "isn": {
+ "type": "string"
+ },
+ "risn": {
+ "type": "string"
+ },
+ "tag": {
+ "type": "string"
+ },
+ "rtag": {
+ "type": "string"
+ },
+ "pkt": {
+ "type": "string"
+ },
+ "oct": {
+ "type": "string"
+ },
+ "rpkt": {
+ "type": "string"
+ },
+ "roct": {
+ "type": "string"
+ },
+ "app": {
+ "type": "string"
+ },
+ "end-reason": {
+ "type": "string"
}
- }
- }
-}
\ No newline at end of file
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/kafka-client/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/kafka-client/tasks/main.yml b/deployment/roles/kafka-client/tasks/main.yml
new file mode 100644
index 0000000..1674225
--- /dev/null
+++ b/deployment/roles/kafka-client/tasks/main.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Retrieve HDP repository definition
+ get_url:
+ url: "{{ hdp_repo_def }}"
+ dest: /etc/yum.repos.d/hdp.repo
+ mode: 0644
+
+ - name: Install kafka
+ yum:
+ name: kafka
+ register: result
+ until: result.rc == 0
+ retries: 5
+ delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/metron_streaming/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/defaults/main.yml b/deployment/roles/metron_streaming/defaults/main.yml
index 9c67a24..3e96705 100644
--- a/deployment/roles/metron_streaming/defaults/main.yml
+++ b/deployment/roles/metron_streaming/defaults/main.yml
@@ -32,5 +32,5 @@ storm_topologies:
- "{{ metron_directory }}/config/topologies/bro/remote.yaml"
- "{{ metron_directory }}/config/topologies/snort/remote.yaml"
- "{{ metron_directory }}/config/topologies/yaf/remote.yaml"
- - "{{ metron_directory }}/config/topologies/pcap/parse.yaml"
+ - "{{ metron_directory }}/config/topologies/pcap/remote.yaml"
- "{{ metron_directory }}/config/topologies/enrichment/remote.yaml"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/metron_ui/defaults/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/defaults/main.yml b/deployment/roles/metron_ui/defaults/main.yml
new file mode 100644
index 0000000..23aed40
--- /dev/null
+++ b/deployment/roles/metron_ui/defaults/main.yml
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+metron_version: 0.1BETA
+metron_directory: /usr/metron/{{ metron_version }}
+metron_ui_directory: "{{ metron_directory }}/metron-ui"
+metron_temp_archive: /tmp/metron-ui.tar.gz
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/metron_ui/tasks/copy-source.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/copy-source.yml b/deployment/roles/metron_ui/tasks/copy-source.yml
new file mode 100644
index 0000000..0487cf2
--- /dev/null
+++ b/deployment/roles/metron_ui/tasks/copy-source.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Verify {{ metron_ui_directory }} exists
+ file:
+ path: "{{ metron_ui_directory }}"
+ state: directory
+ mode: 0755
+ owner: root
+ group: root
+
+- name: Archive metron-ui on localhost
+ shell: tar -czf {{ metron_temp_archive }} .
+ args:
+ chdir: "{{ playbook_dir }}/../../metron-ui"
+ creates: "{{ metron_temp_archive }}"
+ warn: false #Warns to use unarchive - unarchive does not archive
+ become: false
+ delegate_to: localhost
+ run_once: true
+
+- name: Extract metron-ui tarball
+ unarchive:
+ src: "{{ metron_temp_archive }}"
+ dest: "{{ metron_ui_directory }}"
+ creates: "{{ metron_ui_directory}}/config"
+
+- name: Delete {{ metron_temp_archive }}
+ local_action: file path="{{ metron_temp_archive }}" state=absent
+ become: false
+ run_once: true
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/main.yml b/deployment/roles/metron_ui/tasks/main.yml
index 01ffc17..fd3422b 100644
--- a/deployment/roles/metron_ui/tasks/main.yml
+++ b/deployment/roles/metron_ui/tasks/main.yml
@@ -17,27 +17,23 @@
---
- name: Install Metron UI dependencies
yum:
- pkg: "{{ item.package }}"
+ pkg: "{{ item }}"
state: installed
with_items:
- - { package: 'libpcap-devel'}
- - { package: 'wireshark'}
- - { package: 'nodejs'}
- - { package: 'npm'}
+ - libpcap-devel
+ - wireshark
+ - nodejs
+ - npm
register: result
until: result.rc == 0
retries: 5
delay: 10
-
-- name: Copy Metron UI source
- copy:
- src: "{{ playbook_dir }}/../../metron-ui"
- dest: "{{ metron_directory }}"
+- include: copy-source.yml
- name: Configure Metron UI
lineinfile:
- dest="{{ metron_directory }}/metron-ui/config.json"
+ dest="{{ metron_ui_directory }}/config.json"
regexp="{{ item.regexp }}"
line="{{ item.line }}"
state=present
@@ -47,11 +43,9 @@
- name: Install Node dependencies
npm:
- name: "{{ item }}"
+ name: pm2
path: "{{ metron_ui_directory }}"
global: true
- with_items:
- - pm2
- name: Install Metron UI
npm:
@@ -59,4 +53,4 @@
production: no
- name: Start Metron UI
- shell: "pm2 start /usr/metron/0.1BETA/metron-ui/lib/metron-ui.js --name metron"
+ shell: "pm2 start {{ metron_ui_directory }}/lib/metron-ui.js --name metron"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/metron_ui/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/vars/main.yml b/deployment/roles/metron_ui/vars/main.yml
deleted file mode 100644
index ea41dfa..0000000
--- a/deployment/roles/metron_ui/vars/main.yml
+++ /dev/null
@@ -1,22 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-metron_version: 0.1BETA
-metron_directory: /usr/metron/{{ metron_version }}
-metron_ui_directory: "{{ metron_directory }}/metron-ui"
-metron_jar_url: http://192.168.1.174:8080/repository/internal/com/opensoc/OpenSOC-Topologies/{{ metron_version }}/OpenSOC-Topologies-{{ metron_version }}.jar
-metron_source_url: https://github.com/merrimanr/incubator-metron/archive/master.zip
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/pycapa/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/meta/main.yml b/deployment/roles/pycapa/meta/main.yml
index 85c04df..3aaa18d 100644
--- a/deployment/roles/pycapa/meta/main.yml
+++ b/deployment/roles/pycapa/meta/main.yml
@@ -19,3 +19,4 @@ dependencies:
- ambari_gather_facts
- epel
- python-pip
+ - kafka-client
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/snort/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/meta/main.yml b/deployment/roles/snort/meta/main.yml
index df99998..f742973 100644
--- a/deployment/roles/snort/meta/main.yml
+++ b/deployment/roles/snort/meta/main.yml
@@ -20,3 +20,5 @@ dependencies:
- epel
- libselinux-python
- build-tools
+ - kafka-client
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/yaf/meta/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/meta/main.yml b/deployment/roles/yaf/meta/main.yml
index 324f2ff..ff366b8 100644
--- a/deployment/roles/yaf/meta/main.yml
+++ b/deployment/roles/yaf/meta/main.yml
@@ -20,3 +20,4 @@ dependencies:
- build-tools
- java_jdk
- libselinux-python
+ - kafka-client
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/yaf/tasks/kafka-client.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/kafka-client.yml b/deployment/roles/yaf/tasks/kafka-client.yml
deleted file mode 100644
index 1674225..0000000
--- a/deployment/roles/yaf/tasks/kafka-client.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
- - name: Retrieve HDP repository definition
- get_url:
- url: "{{ hdp_repo_def }}"
- dest: /etc/yum.repos.d/hdp.repo
- mode: 0644
-
- - name: Install kafka
- yum:
- name: kafka
- register: result
- until: result.rc == 0
- retries: 5
- delay: 10
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
index b583393..15f67f6 100644
--- a/deployment/roles/yaf/tasks/main.yml
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -15,6 +15,5 @@
# limitations under the License.
#
---
-- include: kafka-client.yml
- include: fixbuf.yml
- include: yaf.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/deployment/vagrant/singlenode-vagrant/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/vagrant/singlenode-vagrant/ansible.cfg b/deployment/vagrant/singlenode-vagrant/ansible.cfg
index 9c650c2..0a75321 100644
--- a/deployment/vagrant/singlenode-vagrant/ansible.cfg
+++ b/deployment/vagrant/singlenode-vagrant/ansible.cfg
@@ -20,3 +20,4 @@ host_key_checking = false
library = ../../extra_modules
roles_path = ../../roles
pipelining = True
+log_path = ./ansible.log
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/63432776/metron-streaming/Metron-DataLoads/pom.xml
----------------------------------------------------------------------
diff --git a/metron-streaming/Metron-DataLoads/pom.xml b/metron-streaming/Metron-DataLoads/pom.xml
index 5cac558..4e7f76b 100644
--- a/metron-streaming/Metron-DataLoads/pom.xml
+++ b/metron-streaming/Metron-DataLoads/pom.xml
@@ -79,7 +79,6 @@
<groupId>org.mitre.taxii</groupId>
<artifactId>taxii</artifactId>
<version>1.1.0.1</version>
- <!--scope>provided</scope-->
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
@@ -242,7 +241,6 @@
<shadedPattern>org.apache.metron.httpcore.dataload</shadedPattern>
</relocation>
</relocations>
- <!--minimizeJar>true</minimizeJar-->
<artifactSet>
<excludes>
<exclude>classworlds:classworlds</exclude>