You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/02/24 22:17:29 UTC
[2/2] incubator-metron git commit: METRON-43 Refactor Deployment
Scripts for Correctness and Ease of Use (dlyle65535 via cestella) closes
apache/incubator-metron#28
METRON-43 Refactor Deployment Scripts for Correctness and Ease of Use (dlyle65535 via cestella) closes apache/incubator-metron#28
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/5fc17e63
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/5fc17e63
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/5fc17e63
Branch: refs/heads/master
Commit: 5fc17e63c522b46f257ca503de8e062a7458667c
Parents: 775a755
Author: dlyle65535 <dl...@gmail.com>
Authored: Wed Feb 24 16:17:12 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Wed Feb 24 16:17:12 2016 -0500
----------------------------------------------------------------------
deployment/ansible.cfg | 19 +++++
.../inventory/metron_example/group_vars/all | 59 ++++++++-----
deployment/inventory/metron_example/hosts | 67 ++++++++++-----
.../inventory/multinode-vagrant/group_vars/all | 55 ++++++++++++
deployment/inventory/multinode-vagrant/hosts | 59 +++++++++++++
.../inventory/singlenode-vagrant/group_vars/all | 53 +++++++-----
deployment/inventory/singlenode-vagrant/hosts | 37 +++++---
deployment/playbooks/metron_install.yml | 17 +++-
deployment/roles/ambari_common/README.md | 38 ---------
deployment/roles/ambari_common/tasks/main.yml | 38 +++++----
.../ambari_common/tasks/passwd_less_ssh.yml | 27 +++---
deployment/roles/ambari_common/vars/main.yml | 2 -
deployment/roles/ambari_config/README.md | 38 ---------
deployment/roles/ambari_config/tasks/main.yml | 2 +-
deployment/roles/ambari_config/vars/main.yml | 17 ----
.../vars/multi_vagrant_cluster.yml | 80 +++++++++++++++++
.../roles/ambari_config/vars/single_node_vm.yml | 13 +--
.../roles/ambari_config/vars/small_cluster.yml | 37 +++++---
.../roles/ambari_gather_facts/tasks/main.yml | 20 ++---
deployment/roles/ambari_master/README.md | 38 ---------
deployment/roles/ambari_master/tasks/main.yml | 4 +-
deployment/roles/ambari_slave/README.md | 38 ---------
deployment/roles/ambari_slave/files/hostname.sh | 19 +++++
deployment/roles/ambari_slave/tasks/main.yml | 16 +++-
deployment/roles/ambari_slave/vars/main.yml | 1 -
.../elasticsearch/files/elasticsearch.repo | 25 +++---
deployment/roles/elasticsearch/tasks/main.yml | 12 ++-
deployment/roles/hadoop_setup/tasks/main.yml | 2 +
deployment/roles/metron_common/tasks/main.yml | 47 ++++++++++
deployment/roles/metron_common/vars/main.yml | 19 +++++
deployment/roles/metron_pcapservice/README.md | 38 ---------
.../metron_pcapservice/tasks/hdfs-site.yml | 30 +++++++
.../roles/metron_pcapservice/tasks/main.yml | 25 +++++-
.../roles/metron_streaming/tasks/main.yml | 9 --
deployment/roles/metron_streaming/vars/main.yml | 4 +-
deployment/roles/metron_ui/tasks/main.yml | 2 +-
deployment/roles/mysql/files/geoip_ddl.sql | 49 +++++++++++
deployment/roles/mysql/files/mylogin.cnf | 19 +++++
.../mysql57-community-release-el6-7.noarch.rpm | Bin 0 -> 8848 bytes
deployment/roles/mysql/handlers/main.yml | 19 +++++
deployment/roles/mysql/tasks/main.yml | 85 +++++++++++++++++++
deployment/roles/mysql/templates/.my.cnf | 20 +++++
deployment/roles/mysql/vars/main.yml | 20 +++++
deployment/roles/pcap_replay/files/pcap-replay | 27 +++---
deployment/roles/pcap_replay/tasks/main.yml | 1 +
deployment/roles/pycapa/tasks/main.yml | 4 +-
deployment/roles/snort/tasks/snort.yml | 28 ++++++
deployment/roles/yaf/tasks/main.yml | 27 ++++++
deployment/roles/yaf/vars/main.yml | 1 +
.../vagrant/multinode-vagrant/Vagrantfile | 13 +--
pom.xml | 3 +-
51 files changed, 919 insertions(+), 404 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/ansible.cfg b/deployment/ansible.cfg
index 05de078..9b3916b 100644
--- a/deployment/ansible.cfg
+++ b/deployment/ansible.cfg
@@ -1,4 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
[defaults]
host_key_checking = false
library = extra_modules
roles_path = ./roles
+
+[ssh_connection]
+control_path = %(directory)s/%%h-%%p-%%r
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/inventory/metron_example/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/group_vars/all b/deployment/inventory/metron_example/group_vars/all
index a8b5bff..e72888c 100644
--- a/deployment/inventory/metron_example/group_vars/all
+++ b/deployment/inventory/metron_example/group_vars/all
@@ -1,32 +1,30 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
-# metron variables
-metron_version: 0.1BETA
-java_home: /usr/jdk64/jdk1.8.0_40
+#Ansible Variables
+ansible_ssh_private_key_file: /Path/to/private/key/file #Change This
+ansible_ssh_user: root
-# ambari
-ambari_host: metron01.cloud.hortonworks.com
+#Ambari variables
+ambari_host: "{{ groups.ambari_master[0] }}"
ambari_port: 8080
ambari_user: admin
ambari_password: admin
-cluster_type: single_node_vm
-hdp_host_group: ["metron01.cloud.hortonworks.com"]
+cluster_type: small_cluster
# hbase
pcap_hbase_table: pcap
@@ -39,7 +37,24 @@ bro_topic: bro
yaf_topic: ipfix
snort_topic: snort
-# other values
+#elasticsearch
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth0
elasticsearch_web_port: 9200
+
+# metron variables
+metron_version: 0.1BETA
+java_home: /usr/jdk64/jdk1.8.0_40
pcapservice_port: 8081
-sniff_interface: eth1
+sniff_interface: eth0
+bro_version: "2.4.1"
+fixbuf_version: "1.7.1"
+yaf_version: "2.8.0"
+daq_version: "2.0.6-1"
+iface: "eth0"
+pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+pycapa_home: "/opt/pycapa"
+snort_version: "2.9.8.0-1"
+snort_alert_csv_path: "/var/log/snort/alert.csv"
+pcap_replay: True
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/inventory/metron_example/hosts
----------------------------------------------------------------------
diff --git a/deployment/inventory/metron_example/hosts b/deployment/inventory/metron_example/hosts
index 2aa75cc..0d01327 100644
--- a/deployment/inventory/metron_example/hosts
+++ b/deployment/inventory/metron_example/hosts
@@ -1,38 +1,63 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
[ambari_master]
-host
+node1
+#minimum of 3 - 6 from 12 node cluser
[ambari_slave]
-host
+node2
+node3
+node4
+node5
+node6
+node7
+node8
+#last ambari_slave
[hadoop_client]
-host
+node9
+#3rd ambari_slave
[enrichment]
-host ansible_ssh_private_key_file=path_to_key_file ansible_ssh_user=user
+node1
+#1 or more
[search]
-host ansible_ssh_private_key_file=path_to_key_file ansible_ssh_user=user
+node10
+node11
+node12
+#1 only
+[sensors]
+node1
+
+#same as mysql in 12 node topology
[web]
-host ansible_ssh_private_key_file=path_to_key_file ansible_ssh_user=user
+node12
+
+[mysql]
+node12
+
+[metron:children]
+enrichment
+search
+web
+sensors
+mysql
+hadoop_client
-[sensors]
-host ansible_ssh_private_key_file=path_to_key_file ansible_ssh_user=user
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/inventory/multinode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/group_vars/all b/deployment/inventory/multinode-vagrant/group_vars/all
new file mode 100644
index 0000000..2f7b10f
--- /dev/null
+++ b/deployment/inventory/multinode-vagrant/group_vars/all
@@ -0,0 +1,55 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#Ambari variables
+ambari_host: "{{ groups.ambari_master[0] }}"
+hdp_host_group: "{{ groups.ambari_slave }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
+cluster_type: multi_vagrant_cluster
+
+# hbase
+pcap_hbase_table: pcap
+tracker_hbase_table: access_tracker
+threatintel_ip_hbase_table: malicious_ip
+
+# kafka
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: ipfix
+snort_topic: snort
+
+#elasticsearch
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
+elasticsearch_web_port: 9200
+
+# metron variables
+metron_version: 0.1BETA
+java_home: /usr/jdk64/jdk1.8.0_40
+pcapservice_port: 8081
+sniff_interface: eth1
+bro_version: "2.4.1"
+fixbuf_version: "1.7.1"
+yaf_version: "2.8.0"
+daq_version: "2.0.6-1"
+iface: "eth0"
+pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+pycapa_home: "/opt/pycapa"
+snort_version: "2.9.8.0-1"
+snort_alert_csv_path: "/var/log/snort/alert.csv"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/inventory/multinode-vagrant/hosts
----------------------------------------------------------------------
diff --git a/deployment/inventory/multinode-vagrant/hosts b/deployment/inventory/multinode-vagrant/hosts
new file mode 100644
index 0000000..d84ab1e
--- /dev/null
+++ b/deployment/inventory/multinode-vagrant/hosts
@@ -0,0 +1,59 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[ambari_master]
+node1
+
+#minimum of 3 - 6 from 12 node cluser
+[ambari_slave]
+node2
+node3
+node4
+
+#last ambari_slave
+[hadoop_client]
+node4
+
+[enrichment]
+node4
+
+#1 or more
+[search]
+node1
+
+#1 only
+[sensors]
+node1
+
+#same as mysql in 12 node topology
+[web]
+node3
+
+[mysql]
+node3
+
+[metron:children]
+enrichment
+search
+web
+sensors
+mysql
+hadoop_client
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/inventory/singlenode-vagrant/group_vars/all
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/group_vars/all b/deployment/inventory/singlenode-vagrant/group_vars/all
index 79f2d07..efef4e5 100644
--- a/deployment/inventory/singlenode-vagrant/group_vars/all
+++ b/deployment/inventory/singlenode-vagrant/group_vars/all
@@ -1,32 +1,27 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
-# metron variables
-metron_version: 0.1BETA
-java_home: /usr/jdk64/jdk1.8.0_40
-
-# ambari
-ambari_host: node1
+#Ambari variables
+ambari_host: "{{ groups.ambari_master[0] }}"
+hdp_host_group: "{{ groups.ambari_slave }}"
ambari_port: 8080
ambari_user: admin
ambari_password: admin
cluster_type: single_node_vm
-hdp_host_group: ["node1"]
# hbase
pcap_hbase_table: pcap
@@ -39,7 +34,23 @@ bro_topic: bro
yaf_topic: ipfix
snort_topic: snort
-# other values
+#elasticsearch
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth1
elasticsearch_web_port: 9200
+
+# metron variables
+metron_version: 0.1BETA
+java_home: /usr/jdk64/jdk1.8.0_40
pcapservice_port: 8081
sniff_interface: eth1
+bro_version: "2.4.1"
+fixbuf_version: "1.7.1"
+yaf_version: "2.8.0"
+daq_version: "2.0.6-1"
+iface: "eth0"
+pycapa_repo: "https://github.com/OpenSOC/pycapa.git"
+pycapa_home: "/opt/pycapa"
+snort_version: "2.9.8.0-1"
+snort_alert_csv_path: "/var/log/snort/alert.csv"
+pcap_replay: True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/inventory/singlenode-vagrant/hosts
----------------------------------------------------------------------
diff --git a/deployment/inventory/singlenode-vagrant/hosts b/deployment/inventory/singlenode-vagrant/hosts
index cc31bb1..6fd8b18 100644
--- a/deployment/inventory/singlenode-vagrant/hosts
+++ b/deployment/inventory/singlenode-vagrant/hosts
@@ -1,20 +1,20 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
+
[ambari_master]
node1
@@ -35,3 +35,14 @@ node1
[sensors]
node1
+
+[mysql]
+node1
+
+[metron:children]
+enrichment
+search
+web
+sensors
+mysql
+hadoop_client
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index 6873b78..ad070c9 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -15,6 +15,11 @@
# limitations under the License.
#
---
+- hosts: metron
+ sudo: yes
+ roles:
+ - role: metron_common
+
- hosts: hadoop_client
sudo: yes
roles:
@@ -23,11 +28,20 @@
- hosts: search
sudo: yes
+ vars:
+ es_hosts: "{% set comma = joiner(',') %}{% for host in groups['search'] -%}{{ comma() }}{{ host }}{%- endfor %}"
roles:
- role: elasticsearch
tags:
- elasticsearch
+- hosts: mysql
+ sudo: yes
+ roles:
+ - role: mysql
+ tags:
+ - mysql
+
- hosts: sensors
sudo: yes
roles:
@@ -37,7 +51,7 @@
- role: bro
- role: snort
- role: yaf
- - role: pcap_replay
+ - { role: pcap_replay , when: pcap_replay | default(False) == True }
tags:
- sensors
@@ -57,3 +71,4 @@
- role: metron_pcapservice
tags:
- web
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_common/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/README.md b/deployment/roles/ambari_common/README.md
deleted file mode 100644
index 225dd44..0000000
--- a/deployment/roles/ambari_common/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-Role Name
-=========
-
-A brief description of the role goes here.
-
-Requirements
-------------
-
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
-
-Role Variables
---------------
-
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
-
-Dependencies
-------------
-
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
- - hosts: servers
- roles:
- - { role: username.rolename, x: 42 }
-
-License
--------
-
-BSD
-
-Author Information
-------------------
-
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/tasks/main.yml b/deployment/roles/ambari_common/tasks/main.yml
index ec5576b..992468e 100644
--- a/deployment/roles/ambari_common/tasks/main.yml
+++ b/deployment/roles/ambari_common/tasks/main.yml
@@ -15,12 +15,28 @@
# limitations under the License.
#
---
-# tasks file for ambari_common
+- name: Check OS Version
+ fail: msg="Ambari HDP deployment supports CentOS 6 only."
+ when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
+
- name: Install libselinux-python
yum: name=libselinux-python
- include: passwd_less_ssh.yml
+- name: upgrade all packages
+ yum: name=* state=latest
+
+- name: install ntp
+ yum: name=ntp state=present
+
+- name: Ensure ntp is running and enabled
+ service: name=ntpd state=started enabled=yes
+
+- name: Ensure iptables is stopped and is not running at boot time.
+ ignore_errors: yes
+ service: name=iptables state=stopped enabled=no
+
#
# ambari uses socket.getfqdn() to find the hostname. with 'localhost.localdomain'
# in '/etc/hosts' this function will report the hostname as 'localhost.localdomain'
@@ -37,22 +53,14 @@
- name: Add localhost to /etc/hosts
lineinfile: dest=/etc/hosts line="127.0.0.1 localhost"
+- name: get epel-repo rpm
+ get_url: dest=/tmp/epel-release.rpm url=http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+- name: install epel-repo rpm
+ yum: pkg=/tmp/epel-release.rpm state=installed
+
- name: Download HDP repo
get_url: url="{{ rhel_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
-- name: Install EPEL repo.
- yum:
- name: "{{ epel_repo_url }}"
- state: present
-
- name: Clean yum
shell: yum clean all
-
-- name: install ntp
- yum: name=ntp state=present
-
-- name: Ensure ntp is running and enabled
- service: name=ntpd state=started enabled=yes
-
-- name: Ensure iptables is stopped and is not running at boot time.
- service: name=iptables state=stopped enabled=no
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml b/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
index 67b07e9..bb92f4e 100644
--- a/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
+++ b/deployment/roles/ambari_common/tasks/passwd_less_ssh.yml
@@ -18,21 +18,16 @@
- name: Generate ssh key pair for "{{ ambari_user }}"
user: name={{ ambari_user }} generate_ssh_key=yes
-- name: Fetch the generated public key from "{{ ansible_hostname }}"
- fetch: src=~{{ ambari_user }}/.ssh/id_rsa.pub dest=/tmp/keys/{{ ansible_hostname }}.pub flat=yes
+- name: Fetch the generated public key
+ fetch: src=~{{ ambari_user }}/.ssh/id_rsa.pub dest=/tmp/keys/{{ inventory_hostname }}.pub flat=yes
-- name: Set ownership/permissions on 'authorized_keys' file
- file:
- path: "~{{ ambari_user }}/.ssh/authorized_keys"
- owner: "{{ ambari_user }}"
- group: "{{ ambari_user }}"
- mode: 0644
- state: touch
+- name: Add key pairs to existing authorized_keys
+ authorized_key: user={{ ambari_user }} key="{{ lookup('file', '/tmp/keys/{{ item }}.pub') }}"
+ with_items:
+ play_hosts
+
+#ambari_common.hosts
+- name: Remove local copy of ssh keys
+ local_action: file path=/tmp/keys/{{ inventory_hostname }}.pub state=absent
+ sudo: False
-- name: Assemble each host's public key into an 'authorized_keys' file
- assemble:
- remote_src: false
- src: /tmp/keys
- dest: "~{{ ambari_user }}/.ssh/authorized_keys"
- owner: "{{ ambari_user }}"
- mode: 0644
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_common/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_common/vars/main.yml b/deployment/roles/ambari_common/vars/main.yml
index f86f9f6..699dcf7 100644
--- a/deployment/roles/ambari_common/vars/main.yml
+++ b/deployment/roles/ambari_common/vars/main.yml
@@ -15,8 +15,6 @@
# limitations under the License.
#
---
-# vars file for ambari_common
-epel_repo_url: "http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm"
rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.2.1/ambari.repo"
ambari_user: "root"
local_tmp_keygen_file: "/tmp/id_rsa.tmp"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_config/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/README.md b/deployment/roles/ambari_config/README.md
deleted file mode 100644
index 225dd44..0000000
--- a/deployment/roles/ambari_config/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-Role Name
-=========
-
-A brief description of the role goes here.
-
-Requirements
-------------
-
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
-
-Role Variables
---------------
-
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
-
-Dependencies
-------------
-
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
- - hosts: servers
- roles:
- - { role: username.rolename, x: 42 }
-
-License
--------
-
-BSD
-
-Author Information
-------------------
-
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_config/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/tasks/main.yml b/deployment/roles/ambari_config/tasks/main.yml
index 73e146d..c351825 100644
--- a/deployment/roles/ambari_config/tasks/main.yml
+++ b/deployment/roles/ambari_config/tasks/main.yml
@@ -22,7 +22,7 @@
- name: Create the ambari cluster
ambari_cluster_state:
- host: "{{ ambari_host }}"
+ host: "{{ groups.ambari_master[0] }}"
port: "{{ ambari_port }}"
username: "{{ ambari_user }}"
password: "{{ ambari_password }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_config/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/main.yml b/deployment/roles/ambari_config/vars/main.yml
deleted file mode 100644
index 841d185..0000000
--- a/deployment/roles/ambari_config/vars/main.yml
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml b/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
new file mode 100644
index 0000000..1e2e1e8
--- /dev/null
+++ b/deployment/roles/ambari_config/vars/multi_vagrant_cluster.yml
@@ -0,0 +1,80 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+
+hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
+hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, NODEMANAGER]
+spark_master: [SPARK_JOBHISTORYSERVER]
+storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
+storm_slave: [SUPERVISOR]
+kafka_broker: [KAFKA_BROKER]
+zookeeper_master: [ZOOKEEPER_SERVER]
+hbase_master: [HBASE_MASTER]
+hbase_slave: [HBASE_REGIONSERVER]
+hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
+
+master_1_components: "{{ hadoop_master | union(hadoop_clients) }}"
+master_1_host:
+ - "{{groups.ambari_slave[0]}}"
+master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) }}"
+master_2_host:
+ - "{{groups.ambari_slave[1]}}"
+slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"
+
+cluster_name: "metron"
+blueprint_name: "metron_blueprint"
+
+configurations:
+ - hadoop-env:
+ namenode_heapsize: 1024
+ dtnode_heapsize: 1024
+ - hbase-env:
+ hbase_regionserver_heapsize: 1024
+ hbase_master_heapsize: 1024
+ - yarn-env:
+ nodemanager_heapsize: 512
+ yarn_heapsize: 512
+ apptimelineserver_heapsize : 512
+ - mapred-env:
+ jobhistory_heapsize: 256
+ - yarn-site:
+ yarn.nodemanager.resource.memory-mb: 1024
+ yarn.scheduler.maximum-allocation-mb: 1024
+ - storm-site:
+ supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+ - kafka-env:
+ content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
+
+blueprint:
+ stack_name: HDP
+ stack_version: 2.3
+ groups:
+ - name : master_1
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ master_1_components }}"
+ hosts: "{{ master_1_host }}"
+ - name : master_2
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ master_2_components }}"
+ hosts: "{{ master_2_host }}"
+ - name: slaves
+ cardinality: 1+
+ configuration: [] # configuration not yet implemented
+ components: "{{ slave_components }}"
+ hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_config/vars/single_node_vm.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/single_node_vm.yml b/deployment/roles/ambari_config/vars/single_node_vm.yml
index d87d55b..abae6a4 100644
--- a/deployment/roles/ambari_config/vars/single_node_vm.yml
+++ b/deployment/roles/ambari_config/vars/single_node_vm.yml
@@ -26,11 +26,10 @@ storm_slave: [SUPERVISOR]
kafka_broker: [KAFKA_BROKER]
zookeeper_master: [ZOOKEEPER_SERVER]
zookeeper_slave: [ZOOKEEPER_CLIENT]
-ganglia_master: [GANGLIA_SERVER]
hbase_master: [HBASE_MASTER, HBASE_CLIENT]
hbase_slave: [HBASE_REGIONSERVER]
-metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(spark_master) | union(ganglia_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(spark_slave) | union(kafka_broker) | union(hbase_slave) }}"
+metron_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_slave) | union(zookeeper_slave) | union(storm_slave) | union(spark_slave) | union(kafka_broker) | union(hbase_slave) }}"
cluster_name: "metron_cluster"
blueprint_name: "metron_blueprint"
@@ -43,14 +42,18 @@ configurations:
hbase_regionserver_heapsize: 1024
hbase_master_heapsize: 1024
- yarn-env:
- nodemanager_heapsize: 1024
- yarn_heapsize: 1024
+ nodemanager_heapsize: 512
+ yarn_heapsize: 512
+ apptimelineserver_heapsize : 512
+ - mapred-env:
+ jobhistory_heapsize: 256
- yarn-site:
yarn.nodemanager.resource.memory-mb: 1024
yarn.scheduler.maximum-allocation-mb: 1024
- storm-site:
supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
-
+ - kafka-env:
+ content: "{% raw %}\n#!/bin/bash\n\n# Set KAFKA specific environment variables here.\n\n# The java implementation to use.\nexport KAFKA_HEAP_OPTS=\"-Xms256M -Xmx256M\"\nexport KAFKA_JVM_PERFORMANCE_OPTS=\"-server -XX:+UseG1GC -XX:+DisableExplicitGC -Djava.awt.headless=true\"\nexport JAVA_HOME={{java64_home}}\nexport PATH=$PATH:$JAVA_HOME/bin\nexport PID_DIR={{kafka_pid_dir}}\nexport LOG_DIR={{kafka_log_dir}}\nexport KAFKA_KERBEROS_PARAMS={{kafka_kerberos_params}}\n# Add kafka sink to classpath and related depenencies\nif [ -e \"/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\" ]; then\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/ambari-metrics-kafka-sink.jar\n export CLASSPATH=$CLASSPATH:/usr/lib/ambari-metrics-kafka-sink/lib/*\nfi\nif [ -f /etc/kafka/conf/kafka-ranger-env.sh ]; then\n . /etc/kafka/conf/kafka-ranger-env.sh\nfi{% endraw %}"
blueprint:
stack_name: HDP
stack_version: 2.3
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_config/vars/small_cluster.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_config/vars/small_cluster.yml b/deployment/roles/ambari_config/vars/small_cluster.yml
index 46d18ac..592c2dc 100644
--- a/deployment/roles/ambari_config/vars/small_cluster.yml
+++ b/deployment/roles/ambari_config/vars/small_cluster.yml
@@ -15,38 +15,49 @@
# limitations under the License.
#
---
-# vars file for small_cluster blueprint
hadoop_master: [NAMENODE, SECONDARY_NAMENODE, RESOURCEMANAGER, HISTORYSERVER]
-hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, HDFS_CLIENT, NODEMANAGER, YARN_CLIENT, MAPREDUCE2_CLIENT]
+hadoop_slave: [APP_TIMELINE_SERVER, DATANODE, NODEMANAGER]
spark_master: [SPARK_JOBHISTORYSERVER]
-spark_slave: [SPARK_CLIENT]
storm_master: [NIMBUS, STORM_UI_SERVER, DRPC_SERVER]
storm_slave: [SUPERVISOR]
kafka_broker: [KAFKA_BROKER]
zookeeper_master: [ZOOKEEPER_SERVER]
-zookeeper_slave: [ZOOKEEPER_CLIENT]
-ganglia_master: [GANGLIA_SERVER]
-hbase_master: [HBASE_MASTER, HBASE_CLIENT]
+hbase_master: [HBASE_MASTER]
hbase_slave: [HBASE_REGIONSERVER]
+hadoop_clients: [HDFS_CLIENT, YARN_CLIENT, MAPREDUCE2_CLIENT, SPARK_CLIENT, ZOOKEEPER_CLIENT, HBASE_CLIENT]
-master_components: "{{ hadoop_master | union(zookeeper_master) | union(storm_master) | union(spark_master) | union(ganglia_master) | union(hbase_master) }}"
-slave_components: "{{ hadoop_slave | union(zookeeper_slave) | union(storm_slave) | union(spark_slave) | union(kafka_broker) | union(hbase_slave) }}"
+master_1_components: "{{ hadoop_master | union(hadoop_clients) }}"
+master_1_host:
+ - "{{groups.ambari_slave[0]}}"
+master_2_components: "{{ zookeeper_master | union(storm_master) | union(spark_master) | union(hbase_master) | union(hadoop_clients) }}"
+master_2_host:
+ - "{{groups.ambari_slave[1]}}"
+slave_components: "{{ hadoop_slave | union(storm_slave) | union(kafka_broker) | union(hbase_slave) | union(hadoop_clients) }}"
-cluster_name: "metron_cluster"
+cluster_name: "metron"
blueprint_name: "metron_blueprint"
+configurations:
+ - storm-site:
+ supervisor.slots.ports: "[6700, 6701, 6702, 6703]"
+
blueprint:
stack_name: HDP
stack_version: 2.3
groups:
- - name : master
+ - name : master_1
+ cardinality: 1
+ configuration: [] # configuration not yet implemented
+ components: "{{ master_1_components }}"
+ hosts: "{{ master_1_host }}"
+ - name : master_2
cardinality: 1
configuration: [] # configuration not yet implemented
- components: "{{ master_components }}"
- hosts: "{{masters_group}}"
+ components: "{{ master_2_components }}"
+ hosts: "{{ master_2_host }}"
- name: slaves
cardinality: 1+
configuration: [] # configuration not yet implemented
components: "{{ slave_components }}"
- hosts: "{{slaves_group}}"
+ hosts: "{{ groups.ambari_slave | difference(groups.ambari_slave[0]) | difference(groups.ambari_slave[1]) }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_gather_facts/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_gather_facts/tasks/main.yml b/deployment/roles/ambari_gather_facts/tasks/main.yml
index cf01c21..8ec7e40 100644
--- a/deployment/roles/ambari_gather_facts/tasks/main.yml
+++ b/deployment/roles/ambari_gather_facts/tasks/main.yml
@@ -27,7 +27,7 @@
- name: Ambari rest get cluster name
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -39,7 +39,7 @@
- name: Ambari rest get namenode hosts
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/HDFS/components/NAMENODE"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -51,7 +51,7 @@
- name: Ambari rest get namenode core-site tag
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ namenode_host }}/host_components/NAMENODE"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -63,7 +63,7 @@
- name: Ambari rest get namenode core-site properties
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=core-site&tag={{ core_site_tag }}"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -75,7 +75,7 @@
- name: Ambari rest get kafka broker hosts
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/KAFKA/components/KAFKA_BROKER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -87,7 +87,7 @@
- name: Ambari rest get kafka kafka-broker tag
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ kafka_broker_hosts[0] }}/host_components/KAFKA_BROKER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -99,7 +99,7 @@
- name: Ambari rest get kafka kafka-broker properties
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=kafka-broker&tag={{ kafka_broker_tag }}"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -114,7 +114,7 @@
- name: Ambari rest get zookeeper hosts
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/services/ZOOKEEPER/components/ZOOKEEPER_SERVER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -126,7 +126,7 @@
- name: Ambari rest get zookeeper zoo.cfg tag
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/hosts/{{ zookeeper_hosts[0] }}/host_components/ZOOKEEPER_SERVER"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
@@ -138,7 +138,7 @@
- name: Ambari rest get kafka kafka-broker properties
uri:
- url: "http://{{ ambari_host }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
+ url: "http://{{ groups.ambari_master[0] }}:{{ ambari_port }}/api/v1/clusters/{{ cluster_name }}/configurations?type=zoo.cfg&tag={{ zookeeper_tag }}"
user: "{{ ambari_user }}"
password: "{{ ambari_password }}"
force_basic_auth: yes
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_master/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/README.md b/deployment/roles/ambari_master/README.md
deleted file mode 100644
index 225dd44..0000000
--- a/deployment/roles/ambari_master/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-Role Name
-=========
-
-A brief description of the role goes here.
-
-Requirements
-------------
-
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
-
-Role Variables
---------------
-
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
-
-Dependencies
-------------
-
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
- - hosts: servers
- roles:
- - { role: username.rolename, x: 42 }
-
-License
--------
-
-BSD
-
-Author Information
-------------------
-
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_master/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_master/tasks/main.yml b/deployment/roles/ambari_master/tasks/main.yml
index 92c9a5c..8c78f06 100644
--- a/deployment/roles/ambari_master/tasks/main.yml
+++ b/deployment/roles/ambari_master/tasks/main.yml
@@ -27,5 +27,5 @@
- name: start ambari server
service: name=ambari-server state=restarted
-- name : check if ambari-server is up
- wait_for : host={{ ambari_host }} port={{ ambari_port }} delay=10
+- name : check if ambari-server is up on {{ ambari_host }}:{{ambari_port}}
+ wait_for : host={{ ambari_host }} port={{ ambari_port }} delay=10 timeout=120
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_slave/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_slave/README.md b/deployment/roles/ambari_slave/README.md
deleted file mode 100644
index 225dd44..0000000
--- a/deployment/roles/ambari_slave/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-Role Name
-=========
-
-A brief description of the role goes here.
-
-Requirements
-------------
-
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
-
-Role Variables
---------------
-
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
-
-Dependencies
-------------
-
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
- - hosts: servers
- roles:
- - { role: username.rolename, x: 42 }
-
-License
--------
-
-BSD
-
-Author Information
-------------------
-
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_slave/files/hostname.sh
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_slave/files/hostname.sh b/deployment/roles/ambari_slave/files/hostname.sh
new file mode 100644
index 0000000..cc8c1cd
--- /dev/null
+++ b/deployment/roles/ambari_slave/files/hostname.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+echo {{ inventory_hostname }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_slave/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_slave/tasks/main.yml b/deployment/roles/ambari_slave/tasks/main.yml
index c28315a..eac0610 100644
--- a/deployment/roles/ambari_slave/tasks/main.yml
+++ b/deployment/roles/ambari_slave/tasks/main.yml
@@ -19,12 +19,24 @@
- name: Install ambari-agent
yum: name=ambari-agent
+- name: Create ambari-agent hostname script
+ template:
+ src: "../roles/ambari_slave/files/hostname.sh"
+ dest: "/var/lib/ambari-agent/hostname.sh"
+ mode: 0744
+ owner: "{{ ambari_user }}"
+ group: "{{ ambari_user }}"
+
- name: Configure ambari-server hostname in ambari-agent configuration
lineinfile:
dest: /etc/ambari-agent/conf/ambari-agent.ini
- regexp: ^.*hostname=.*$
- line: hostname={{ ansible_hostname }}
+ regexp: "{{ item.regexp }}"
+ line: "{{ item.line }}"
+ insertafter: "{{ item.insertafter }}"
backup: yes
+ with_items:
+ - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
+ - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
- name: Ensure ambari-agent is running
service: name=ambari-agent state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/ambari_slave/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/ambari_slave/vars/main.yml b/deployment/roles/ambari_slave/vars/main.yml
index b3920fc..600b3fc 100644
--- a/deployment/roles/ambari_slave/vars/main.yml
+++ b/deployment/roles/ambari_slave/vars/main.yml
@@ -18,7 +18,6 @@
#
# TODO: duplicates from ambari-common. need all of these moved to group_vars
#
-epel_repo_url: "http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm"
rhel_ambari_install_url: "http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.1.2.1/ambari.repo"
ambari_user: "root"
local_tmp_keygen_file: "/tmp/id_rsa.tmp"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/elasticsearch/files/elasticsearch.repo
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/files/elasticsearch.repo b/deployment/roles/elasticsearch/files/elasticsearch.repo
index e0e4d27..f033ced 100644
--- a/deployment/roles/elasticsearch/files/elasticsearch.repo
+++ b/deployment/roles/elasticsearch/files/elasticsearch.repo
@@ -1,19 +1,18 @@
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#
[elasticsearch-1.7]
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/elasticsearch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/elasticsearch/tasks/main.yml b/deployment/roles/elasticsearch/tasks/main.yml
index 862b6b5..9a08c60 100644
--- a/deployment/roles/elasticsearch/tasks/main.yml
+++ b/deployment/roles/elasticsearch/tasks/main.yml
@@ -15,8 +15,12 @@
# limitations under the License.
#
---
-- name: Alternatives link for "java"
- alternatives: name=java link=/usr/bin/java path=/usr/jdk64/jdk1.8.0_40/bin/java
+
+- name: Install libselinux-python and java
+ yum: name={{item}}
+ with_items:
+ - libselinux-python
+ - java-1.8.0-openjdk
- name: Add Elasticsearch GPG key.
rpm_key:
@@ -40,6 +44,10 @@
state=present
with_items:
- { regexp: '#cluster\.name', line: 'cluster.name: metron' }
+ - { regexp: '#network\.host:', line: 'network.host: _{{
+ elasticsearch_network_interface | default("eth0") }}:ipv4_' }
+ - { regexp: '#discovery\.zen\.ping\.unicast\.hosts',
+ line: 'discovery.zen.ping.unicast.hosts: [ {{ es_hosts }} ]'}
notify: restart elasticsearch
- name: Start Elasticsearch.
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/hadoop_setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/hadoop_setup/tasks/main.yml b/deployment/roles/hadoop_setup/tasks/main.yml
index 0650a70..5e77b99 100644
--- a/deployment/roles/hadoop_setup/tasks/main.yml
+++ b/deployment/roles/hadoop_setup/tasks/main.yml
@@ -15,6 +15,7 @@
# limitations under the License.
#
---
+#must run on hadoop host
- name: Create HBase tables
shell: echo "create '{{ item }}','t'" | hbase shell -n
ignore_errors: yes
@@ -23,6 +24,7 @@
- "{{ tracker_hbase_table }}"
- "{{ threatintel_ip_hbase_table }}"
+#if kafka topic
- name: Create Kafka topics
shell: "{{ kafka_home }}/bin/kafka-topics.sh --zookeeper {{ zookeeper_url }} --create --topic {{ item }} --partitions 1 --replication-factor 1"
ignore_errors: yes
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/tasks/main.yml b/deployment/roles/metron_common/tasks/main.yml
new file mode 100644
index 0000000..29c0821
--- /dev/null
+++ b/deployment/roles/metron_common/tasks/main.yml
@@ -0,0 +1,47 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Check OS Version
+ fail: msg="Metron deployment supports CentOS 6 only."
+ when: (ansible_distribution != "CentOS" or ansible_distribution_major_version != "6")
+
+- name: Check for Metron jar path
+ stat: path={{ metron_jar_path }}
+ register: metron_jars
+
+- name: Verify Metron jars exist
+ fail: msg="Unable to locate staged Metron jars at {{ metron_jar_path }}. Did you run 'mvn package'?"
+ when: metron_jars.stat.exists == True
+
+- name: upgrade all packages
+ yum: name=* state=latest
+
+- name: get epel-repo rpm
+ get_url: dest=/tmp/epel-release.rpm url=http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+- name: install epel-repo rpm
+ yum: pkg=/tmp/epel-release.rpm state=installed
+
+- name: install ntp
+ yum: name=ntp state=present
+
+- name: Ensure ntp is running and enabled
+ service: name=ntpd state=started enabled=yes
+
+- name: Ensure iptables is stopped and is not running at boot time.
+ ignore_errors: yes
+ service: name=iptables state=stopped enabled=no
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_common/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_common/vars/main.yml b/deployment/roles/metron_common/vars/main.yml
new file mode 100644
index 0000000..50aaefd
--- /dev/null
+++ b/deployment/roles/metron_common/vars/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
+metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_pcapservice/README.md
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/README.md b/deployment/roles/metron_pcapservice/README.md
deleted file mode 100644
index 225dd44..0000000
--- a/deployment/roles/metron_pcapservice/README.md
+++ /dev/null
@@ -1,38 +0,0 @@
-Role Name
-=========
-
-A brief description of the role goes here.
-
-Requirements
-------------
-
-Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
-
-Role Variables
---------------
-
-A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
-
-Dependencies
-------------
-
-A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
-
-Example Playbook
-----------------
-
-Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
-
- - hosts: servers
- roles:
- - { role: username.rolename, x: 42 }
-
-License
--------
-
-BSD
-
-Author Information
-------------------
-
-An optional section for the role authors to include contact information, or a website (HTML is not allowed).
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_pcapservice/tasks/hdfs-site.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/hdfs-site.yml b/deployment/roles/metron_pcapservice/tasks/hdfs-site.yml
new file mode 100644
index 0000000..8b435a2
--- /dev/null
+++ b/deployment/roles/metron_pcapservice/tasks/hdfs-site.yml
@@ -0,0 +1,30 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+---
+- name: Fetch hbase-site.xml
+ fetch: src=/etc/hbase/conf/hbase-site.xml dest=/tmp/hbase/conf/hbase-site.xml flat=yes
+ when: not hbase_site.stat.exists
+ delegate_to: "{{ groups.ambari_slave[0] }}"
+
+- name: Create hbase conf directory
+ file: path=/etc/hbase/conf state=directory mode=0755
+ when: not hbase_site.stat.exists
+
+- name: Copy hbase-site.xml
+ copy: src=/tmp/hbase/conf/hbase-site.xml dest=/etc/hbase/conf/hbase-site.xml mode=0644
+ when: not hbase_site.stat.exists
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_pcapservice/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_pcapservice/tasks/main.yml b/deployment/roles/metron_pcapservice/tasks/main.yml
index 9011e7b..949c5a3 100644
--- a/deployment/roles/metron_pcapservice/tasks/main.yml
+++ b/deployment/roles/metron_pcapservice/tasks/main.yml
@@ -26,11 +26,30 @@
src: "{{ pcapservice_jar_path }}"
dest: "{{ metron_directory }}/lib/"
-- name: Alternatives link for "java"
+- name: Check for Java at "{{ java_home }}"
+ stat: path="{{ java_home }}"
+ register: jdk_dir
+
+- name: alternatives link for "java"
alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
with_items:
- - { name: java, link: /usr/bin/java, path: /usr/jdk64/jdk1.8.0_40/bin/java}
- - { name: jar, link: /usr/bin/jar, path: /usr/jdk64/jdk1.8.0_40/bin/jar}
+ - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
+ - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
+ when: jdk_dir.stat.exists
+
+- name: install OpenJdk
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - java-1.8.0-openjdk-devel
+ when: not jdk_dir.stat.exists
+
+- name: Check for hase-site
+ stat: path=/etc/hbase/conf/hbase-site.xml
+ register: hbase_site
+
+- include: hdfs-site.yml
+ when: not hbase_site.stat.exists
- name: Add hbase-site.xml to pcapservice jar
shell: cd /etc/hbase/conf/ && jar -uf {{ metron_directory }}/lib/{{ pcapservice_jar_name }} hbase-site.xml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_streaming/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/tasks/main.yml b/deployment/roles/metron_streaming/tasks/main.yml
index 5eba66d..7d6fe9c 100644
--- a/deployment/roles/metron_streaming/tasks/main.yml
+++ b/deployment/roles/metron_streaming/tasks/main.yml
@@ -22,12 +22,6 @@
- { name: 'bin'}
- { name: 'config'}
-- stat: path={{ metron_jar_path }}
- register: metron_jars
-
-- fail: msg="Unable to locate staged Metron jars at {{ metron_jar_path }}. Did you run 'mvn package'?"
- when: metron_jars.stat.exists == True
-
- name: Copy Metron streaming jars
copy:
src: "{{ metron_jar_path }}"
@@ -50,9 +44,6 @@
- name: Add hbase-site.xml to topology jar
shell: cd {{ hbase_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} hbase-site.xml
-- name: Add elasticsearch.ym to topology jar
- shell: cd {{ elasticsearch_config_path }} && jar -uf {{ metron_directory }}/lib/{{ metron_jar_name }} elasticsearch.yml
-
- name: Copy Metron topology config files
copy:
src: "{{ metron_src_config_path }}/{{ item }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_streaming/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_streaming/vars/main.yml b/deployment/roles/metron_streaming/vars/main.yml
index dbc492b..b93a70f 100644
--- a/deployment/roles/metron_streaming/vars/main.yml
+++ b/deployment/roles/metron_streaming/vars/main.yml
@@ -15,10 +15,10 @@
# limitations under the License.
#
---
-metron_directory: /usr/metron/{{ metron_version }}
metron_jar_name: Metron-Topologies-{{ metron_version }}.jar
-metron_dataloads_name: Metron-DataLoads-{{ metron_version }}-archive.tar.gz
metron_jar_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/target/{{ metron_jar_name }}"
+metron_directory: /usr/metron/{{ metron_version }}
+metron_dataloads_name: Metron-DataLoads-{{ metron_version }}-archive.tar.gz
metron_dataloads_path: "{{ playbook_dir }}/../../metron-streaming/Metron-DataLoads/target/{{ metron_dataloads_name }}"
metron_src_config_path: "{{ playbook_dir }}/../../metron-streaming/Metron-Topologies/src/main/resources/Metron_Configs"
metron_properties_config_path: "{{ metron_directory }}/config/etc/env/config.properties"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/metron_ui/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/metron_ui/tasks/main.yml b/deployment/roles/metron_ui/tasks/main.yml
index 48d5c20..eb2220d 100644
--- a/deployment/roles/metron_ui/tasks/main.yml
+++ b/deployment/roles/metron_ui/tasks/main.yml
@@ -48,7 +48,7 @@
line="{{ item.line }}"
state=present
with_items:
- - { regexp: '"elasticsearch":', line: '" elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
+ - { regexp: '"elasticsearch":', line: '"elasticsearch": { "url": "http://{{ groups.search[0] }}:{{ elasticsearch_web_port }}" },' }
- { regexp: '"pcap":', line: ' "pcap": { "url": "http://{{ groups.web[0] }}:{{ pcapservice_port }}/pcap/pcapGetter","mock": false }' }
- name: Fix pcap_all error
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/files/geoip_ddl.sql
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/geoip_ddl.sql b/deployment/roles/mysql/files/geoip_ddl.sql
new file mode 100644
index 0000000..02616c6
--- /dev/null
+++ b/deployment/roles/mysql/files/geoip_ddl.sql
@@ -0,0 +1,49 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ */
+CREATE DATABASE IF NOT EXISTS GEO;
+
+USE GEO;
+
+DROP TABLE IF EXISTS `blocks`;
+CREATE TABLE `blocks` ( `startIPNum` int(10) unsigned NOT NULL,`endIPNum` int(10) unsigned NOT NULL,`locID`
+int(10) unsigned NOT NULL, PRIMARY KEY (`startIPNum`,`endIPNum`) )
+ENGINE=MyISAM DEFAULT CHARSET=latin1 PACK_KEYS=1 DELAY_KEY_WRITE=1;
+
+DROP TABLE IF EXISTS `location`;
+CREATE TABLE `location` (`locID` int(10) unsigned NOT NULL,`country` char(2) default NULL,`region` char(2)
+ default NULL,`city` varchar(45) default NULL,`postalCode` char(7) default NULL,`latitude` double default
+NULL,`longitude` double default NULL,`dmaCode` char(3) default NULL,`areaCode` char(3) default NULL,PRIMARY KEY
+ (`locID`),KEY `Index_Country` (`country`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 ROW_FORMAT=FIXED;
+
+load data infile '/var/lib/mysql-files/GeoLiteCity-Blocks.csv' into table `blocks` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+load data infile '/var/lib/mysql-files/GeoLiteCity-Location.csv' into table `location` fields terminated by ',' optionally enclosed by '"' lines terminated by '\n' ignore 2 lines;
+
+
+DELIMITER $$
+DROP FUNCTION IF EXISTS `IPTOLOCID` $$
+CREATE FUNCTION `IPTOLOCID`( ip VARCHAR(15)) RETURNS int(10) unsigned
+ BEGIN
+ DECLARE ipn INTEGER UNSIGNED;
+ DECLARE locID_var INTEGER;
+ IF ip LIKE '192.168.%' OR ip LIKE '10.%' THEN RETURN 0;
+ END IF;
+ SET ipn = INET_ATON(ip);
+ SELECT locID INTO locID_var FROM `blocks` INNER JOIN (SELECT MAX(startIPNum) AS start FROM `blocks` WHERE startIPNum <= ipn) AS s ON (startIPNum = s.start) WHERE endIPNum >= ipn;
+ RETURN locID_var;
+ END
+$$
+DELIMITER ;
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/files/mylogin.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/mylogin.cnf b/deployment/roles/mysql/files/mylogin.cnf
new file mode 100644
index 0000000..b8d5781
--- /dev/null
+++ b/deployment/roles/mysql/files/mylogin.cnf
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+[client]
+user=root
+password=P@ssw0rd
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm b/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm
new file mode 100644
index 0000000..8603602
Binary files /dev/null and b/deployment/roles/mysql/files/mysql57-community-release-el6-7.noarch.rpm differ
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/handlers/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/handlers/main.yml b/deployment/roles/mysql/handlers/main.yml
new file mode 100644
index 0000000..112c5ca
--- /dev/null
+++ b/deployment/roles/mysql/handlers/main.yml
@@ -0,0 +1,19 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: restart elasticsearch
+ service: name=elasticsearch state=restarted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/tasks/main.yml b/deployment/roles/mysql/tasks/main.yml
new file mode 100644
index 0000000..91db896
--- /dev/null
+++ b/deployment/roles/mysql/tasks/main.yml
@@ -0,0 +1,85 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Create temporary directories
+ file:
+ path: "/tmp/{{ item }}"
+ state: directory
+ mode: 0755
+ with_items:
+ - "geoip"
+
+- name: Distribute Mysql
+ copy:
+ src: "{{ mysql_rpm_version }}.rpm"
+ dest: /tmp
+
+- name: Install Msyql Yum Repository
+ yum:
+ name: "/tmp/{{ mysql_rpm_version }}.rpm"
+
+- name: Install MySQL
+ yum:
+ name: "{{ item }}"
+ state: latest
+ with_items:
+ - "mysql-community-server"
+ - "MySQL-python"
+
+- name: Start MySQL
+ service:
+ name: mysqld
+ state: started
+ enabled: yes
+
+- name: Retrieve temporary root password
+ shell: "grep 'temporary password' /var/log/mysqld.log | sed 's/.*root@localhost: //'"
+ args:
+ creates: ~/.my.cnf
+ register: temp_root_password
+
+- name: Update mysql root password
+ command: "mysqladmin --user=root --password='{{ temp_root_password.stdout }}' password '{{ mysql_root_password }}'"
+ ignore_errors: yes
+ args:
+ creates: ~/.my.cnf
+
+- name: Copy mylogin.cnf
+ copy:
+ src: mylogin.cnf
+ dest: ~/.my.cnf
+
+- name: Download GeoIP databases
+ unarchive:
+ src: http://geolite.maxmind.com/download/geoip/database/GeoLiteCity_CSV/GeoLiteCity-latest.tar.xz
+ dest: /tmp/geoip
+ copy: no
+ creates: /tmp/geopip/*/GeoLiteCity-Blocks.csv
+
+- name: Copy to MySQL import directory
+ shell: "cp /tmp/geoip/*/*.csv /var/lib/mysql-files/"
+
+- name: Copy DDL
+ copy:
+ src: geoip_ddl.sql
+ dest: /tmp/geoip_ddl.sql
+
+- name: Import GeoIP DDL
+ mysql_db:
+ name: all
+ state: import
+ target: /tmp/geoip_ddl.sql
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/templates/.my.cnf
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/templates/.my.cnf b/deployment/roles/mysql/templates/.my.cnf
new file mode 100644
index 0000000..d5c0825
--- /dev/null
+++ b/deployment/roles/mysql/templates/.my.cnf
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[client]
+user=root
+password={{ mysql_root_password }}
+host=localhost
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/mysql/vars/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/mysql/vars/main.yml b/deployment/roles/mysql/vars/main.yml
new file mode 100644
index 0000000..ccf2426
--- /dev/null
+++ b/deployment/roles/mysql/vars/main.yml
@@ -0,0 +1,20 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+mysql_rpm_version: mysql57-community-release-el6-7.noarch
+mysql_root_password: P@ssw0rd
+
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/pcap_replay/files/pcap-replay
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/files/pcap-replay b/deployment/roles/pcap_replay/files/pcap-replay
index 912b7dc..700236d 100644
--- a/deployment/roles/pcap_replay/files/pcap-replay
+++ b/deployment/roles/pcap_replay/files/pcap-replay
@@ -1,21 +1,20 @@
#!/usr/bin/env bash
#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# http://www.apache.org/licenses/LICENSE-2.0
#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
# pcap replay daemon
# chkconfig: 345 20 80
# description: Replays packet capture data stored in libpcap format
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/pcap_replay/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pcap_replay/tasks/main.yml b/deployment/roles/pcap_replay/tasks/main.yml
index 7a9a2d9..842bb84 100644
--- a/deployment/roles/pcap_replay/tasks/main.yml
+++ b/deployment/roles/pcap_replay/tasks/main.yml
@@ -15,6 +15,7 @@
# limitations under the License.
#
---
+
- include: tcpreplay.yml
- include: service.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/pycapa/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/pycapa/tasks/main.yml b/deployment/roles/pycapa/tasks/main.yml
index 4324ced..71e5964 100644
--- a/deployment/roles/pycapa/tasks/main.yml
+++ b/deployment/roles/pycapa/tasks/main.yml
@@ -22,10 +22,12 @@
yum: name={{item}}
with_items:
- python
- - python-pip
- tcpdump
- git
+- name: Install pip
+ easy_install: name=pip state=latest
+
- name: Clone pycapa repo
git: repo={{ pycapa_repo }} dest={{ pycapa_home }}
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/snort/tasks/snort.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/snort/tasks/snort.yml b/deployment/roles/snort/tasks/snort.yml
index 52339fc..b3d6810 100644
--- a/deployment/roles/snort/tasks/snort.yml
+++ b/deployment/roles/snort/tasks/snort.yml
@@ -15,6 +15,34 @@
# limitations under the License.
#
---
+- name: Install development tools
+ yum: name="@Development tools" state=present
+
+- name: Install snort prerequisites
+ yum: name={{ item }}
+ with_items:
+ - libdnet-devel
+ - rpm-build
+ - libpcap
+ - libpcap-devel
+ - pcre
+ - pcre-devel
+ - zlib-devel
+
+- name: Download daq
+ get_url:
+ url: "https://snort.org/downloads/snort/daq-{{ daq_version }}.src.rpm"
+ dest: "/tmp/daq-{{ daq_version }}.src.rpm"
+
+- name: Build daq
+ shell: "rpmbuild --rebuild daq-{{ daq_version }}.src.rpm"
+ args:
+ chdir: /tmp
+ creates: /root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+
+- name: Install daq
+ yum: name=/root/rpmbuild/RPMS/x86_64/daq-{{ daq_version }}.x86_64.rpm
+
- name: Download snort
get_url:
url: "https://snort.org/downloads/snort/snort-{{ snort_version }}.src.rpm"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/5fc17e63/deployment/roles/yaf/tasks/main.yml
----------------------------------------------------------------------
diff --git a/deployment/roles/yaf/tasks/main.yml b/deployment/roles/yaf/tasks/main.yml
index 09215f9..1e1194d 100644
--- a/deployment/roles/yaf/tasks/main.yml
+++ b/deployment/roles/yaf/tasks/main.yml
@@ -34,6 +34,15 @@
- name: Turn on promiscuous mode for {{ sniff_interface }}
shell: "ip link set {{ sniff_interface }} promisc on"
+- name: Retrieve HDP repository definition
+ get_url:
+ url: "{{ hdp_repo_def }}"
+ dest: /etc/yum.repos.d/hdp.repo
+ mode: 0644
+
+- name: Install kafka
+ yum: name=kafka
+
- set_fact:
yaf: /usr/local/bin/yaf
yafscii: /usr/local/bin/yafscii
@@ -41,6 +50,24 @@
yaf_lock: /var/lock/subsys/yaf
kafka_prod: /usr/hdp/current/kafka-broker/bin/kafka-console-producer.sh
+- name: Check for Java at "{{ java_home }}"
+ stat: path="{{ java_home }}"
+ register: jdk_dir
+
+- name: alternatives link for "java"
+ alternatives: name={{ item.name }} link={{ item.link }} path={{ item.path }}
+ with_items:
+ - { name: java, link: /usr/bin/java, path: "{{ java_home }}/bin/java" }
+ - { name: jar, link: /usr/bin/jar, path: "{{ java_home }}/bin/jar" }
+ when: jdk_dir.stat.exists
+
+- name: install OpenJdk
+ yum: name={{item}}
+ with_items:
+ - java-1.8.0-openjdk
+ - java-1.8.0-openjdk-devel
+ when: not jdk_dir.stat.exists
+
- name: Start yaf
shell: "daemonize -c {{ yaf_home }} -e {{ yaf_log }} -o {{ yaf_log }} -l {{ yaf_lock }} {{ yaf }} --in {{ sniff_interface }} --live pcap | {{ yafscii }} --tabular | {{ kafka_prod }} --broker-list {{ kafka_broker_url }} --topic {{ yaf_topic }}"
args: