You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ni...@apache.org on 2019/05/01 14:50:17 UTC
[metron] branch master updated: METRON-2094 Create CentOS 7
Development Environment (nickwallen) closes apache/metron#1395
This is an automated email from the ASF dual-hosted git repository.
nickallen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metron.git
The following commit(s) were added to refs/heads/master by this push:
new 024aa20 METRON-2094 Create CentOS 7 Development Environment (nickwallen) closes apache/metron#1395
024aa20 is described below
commit 024aa201bc4cbe00af71c57b9e0fe27b51305030
Author: nickwallen <ni...@nickallen.org>
AuthorDate: Wed May 1 10:48:45 2019 -0400
METRON-2094 Create CentOS 7 Development Environment (nickwallen) closes apache/metron#1395
---
.../ansible/playbooks/metron_full_install.yml | 3 -
.../ansible/roles/ambari_common/defaults/main.yml | 3 +-
...ri-repo-centos.yml => ambari-repo-CentOS-6.yml} | 2 +-
...ri-repo-centos.yml => ambari-repo-CentOS-7.yml} | 2 +-
...i-repo-ubuntu.yml => ambari-repo-Ubuntu-14.yml} | 1 -
.../{iptables-centos.yml => firewall-CentOS-6.yml} | 2 +-
.../{iptables-centos.yml => firewall-CentOS-7.yml} | 4 +-
...{iptables-ubuntu.yml => firewall-Ubuntu-14.yml} | 0
.../ansible/roles/ambari_common/tasks/main.yml | 19 +---
.../{iptables-centos.yml => nodejs-CentOS-6.yml} | 8 +-
.../{iptables-centos.yml => nodejs-CentOS-7.yml} | 8 +-
.../{iptables-centos.yml => nodejs-Ubuntu-14.yml} | 8 +-
.../ansible/roles/ambari_common/tasks/nodejs.yml | 36 -------
.../ansible/roles/ambari_config/defaults/main.yml | 5 +
...{dependencies.yml => dependencies-CentOS-6.yml} | 7 +-
...{dependencies.yml => dependencies-CentOS-7.yml} | 7 +-
...dependencies.yml => dependencies-Ubuntu-14.yml} | 5 +-
.../ansible/roles/ambari_config/tasks/main.yml | 2 +-
.../roles/ambari_gather_facts/defaults/main.yml | 4 +
.../ambari_slave/tasks/{main.yml => hostname.yml} | 11 ---
.../tasks/install-agent-CentOS-6.yml} | 4 +-
...nstall-agent.yml => install-agent-CentOS-7.yml} | 12 ++-
.../tasks/install-agent-Ubuntu-14.yml} | 4 +-
.../ansible/roles/ambari_slave/tasks/main.yml | 21 +---
.../ansible/roles/enable-swap/defaults/main.yml | 4 +-
.../enable-swap/tasks/{main.yml => check-swap.yml} | 15 ++-
.../ansible/roles/enable-swap/tasks/main.yml | 23 ++---
.../tasks/{enable-swap.yml => setup-swap.yml} | 9 +-
.../tasks/install-service-CentOS-6.yml} | 8 +-
.../tasks/install-service-CentOS-7.yml} | 20 ++--
.../tasks/install-service-Ubuntu-14.yml} | 8 +-
.../ansible/roles/sensor-stubs/tasks/main.yml | 8 +-
.../templates/sensor-stubs-bro.service | 23 +++++
.../templates/sensor-stubs-snort.service | 23 +++++
.../templates/sensor-stubs-yaf.service | 23 +++++
metron-deployment/development/centos7/Vagrantfile | 108 +++++++++++++++++++++
.../centos7/ansible.cfg} | 17 ++--
.../centos7/ansible/inventory/group_vars/all} | 12 +--
.../centos7/ansible/inventory/hosts} | 35 ++++++-
.../centos7/ansible/playbook.yml} | 12 +--
40 files changed, 346 insertions(+), 180 deletions(-)
diff --git a/metron-deployment/ansible/playbooks/metron_full_install.yml b/metron-deployment/ansible/playbooks/metron_full_install.yml
index ab72b85..c6e8d18 100644
--- a/metron-deployment/ansible/playbooks/metron_full_install.yml
+++ b/metron-deployment/ansible/playbooks/metron_full_install.yml
@@ -33,9 +33,6 @@
- import_playbook: sensor_install.yml
-#
-# deployment report
-#
- hosts: metron
become: false
roles:
diff --git a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
index e7968f5..344638e 100644
--- a/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/defaults/main.yml
@@ -17,5 +17,6 @@
---
hadoop_logrotate_frequency: daily
hadoop_logrotate_retention: 30
-centos_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.6.1.0/ambari.repo
+centos6_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos6/2.x/updates/2.6.1.0/ambari.repo
+centos7_ambari_install_url: http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.1.0/ambari.repo
ubuntu_ambari_repo: http://public-repo-1.hortonworks.com/ambari/ubuntu14/2.x/updates/2.6.1.0
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-6.yml
similarity index 91%
copy from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-6.yml
index 8c1bc33..7e6d9d2 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-6.yml
@@ -16,7 +16,7 @@
#
---
- name: Setup Ambari repo on CentOS
- get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
+ get_url: url="{{ centos6_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
- name: Update package cache on CentOS
yum: name=* update_cache=yes
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-7.yml
similarity index 91%
copy from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-7.yml
index 8c1bc33..f26b39c 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-CentOS-7.yml
@@ -16,7 +16,7 @@
#
---
- name: Setup Ambari repo on CentOS
- get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
+ get_url: url="{{ centos7_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
- name: Update package cache on CentOS
yum: name=* update_cache=yes
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-Ubuntu-14.yml
similarity index 99%
copy from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-Ubuntu-14.yml
index 23c4aca..1e0ca7f 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-Ubuntu-14.yml
@@ -15,7 +15,6 @@
# limitations under the License.
#
---
-
- name: Setup Ambari repo on Ubuntu
shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | tee /etc/apt/sources.list.d/ambari.list
become: true
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-6.yml
similarity index 96%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-6.yml
index 27e67d5..8f01e41 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-6.yml
@@ -15,6 +15,6 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
+- name: Stop iptables on CentOS 6
ignore_errors: yes
service: name=iptables state=stopped enabled=no
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-7.yml
similarity index 90%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-7.yml
index 27e67d5..db77446 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-CentOS-7.yml
@@ -15,6 +15,6 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
+- name: Stop firewalld on CentOS 7
ignore_errors: yes
- service: name=iptables state=stopped enabled=no
+ service: name=firewalld state=stopped enabled=no
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_common/tasks/firewall-Ubuntu-14.yml
similarity index 100%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/firewall-Ubuntu-14.yml
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
index f92a57e..98f5e4d 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/main.yml
@@ -15,21 +15,12 @@
# limitations under the License.
#
---
+- include_tasks: firewall-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml
-- include_tasks: iptables-centos.yml
- when: ansible_distribution == "CentOS"
+- include_tasks: hostname.yml
-- include_tasks: iptables-ubuntu.yml
- when: ansible_distribution == "Ubuntu"
+- include_tasks: ambari-repo-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml
-- include: hostname.yml
+- include_tasks: nodejs-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml
-- include_tasks: ambari-repo-centos.yml
- when: ansible_distribution == "CentOS"
-
-- include_tasks: ambari-repo-ubuntu.yml
- when: ansible_distribution == "Ubuntu"
-
-- include: nodejs.yml
-
-- include: logrotate.yml
+- include_tasks: logrotate.yml
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-6.yml
similarity index 84%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-6.yml
index 27e67d5..8ea7363 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-6.yml
@@ -15,6 +15,8 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
+- name: Install Node.js repo on CentOS 6
+ shell: curl -sL https://rpm.nodesource.com/setup_6.x | bash -
+ args:
+ warn: false
+ become: true
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-7.yml
similarity index 84%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-7.yml
index 27e67d5..b2a2cc0 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-CentOS-7.yml
@@ -15,6 +15,8 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
+- name: Install Node.js repo on CentOS 7
+ shell: curl -sL https://rpm.nodesource.com/setup_7.x | bash -
+ args:
+ warn: false
+ become: true
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-Ubuntu-14.yml
similarity index 84%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
copy to metron-deployment/ansible/roles/ambari_common/tasks/nodejs-Ubuntu-14.yml
index 27e67d5..2bfdecd 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs-Ubuntu-14.yml
@@ -15,6 +15,8 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
+- name: Install Node.js repo on Ubuntu 14
+ shell: curl -sL https://deb.nodesource.com/setup_6.x | bash -
+ args:
+ warn: false
+ become: true
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml b/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
deleted file mode 100644
index ea2067b..0000000
--- a/metron-deployment/ansible/roles/ambari_common/tasks/nodejs.yml
+++ /dev/null
@@ -1,36 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
----
-#
-# for CentOS...
-#
-- name: Install Nodejs repo on CentOS
- shell: curl -sL https://rpm.nodesource.com/setup_6.x | bash -
- args:
- warn: false
- when: ansible_distribution == "CentOS"
- become: true
-
-#
-# for Ubuntu...
-#
-- name: Install Nodejs repo on Ubuntu
- shell: curl -sL https://deb.nodesource.com/setup_6.x | bash -
- args:
- warn: false
- when: ansible_distribution == "Ubuntu"
- become: true
diff --git a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
index ad7ca9e..85d7e90 100644
--- a/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_config/defaults/main.yml
@@ -36,3 +36,8 @@ mapred_reduce_mem_mb : 1229
topology_classpath: '/etc/hbase/conf:/etc/hadoop/conf'
hdp_stack: "2.6"
elasticsearch_network_interface: _site_
+
+ambari_host: "{{ groups.ambari_master[0] }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-6.yml
similarity index 88%
copy from metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
copy to metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-6.yml
index 52d6ca1..72a96d4 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-6.yml
@@ -15,7 +15,12 @@
# limitations under the License.
#
---
-- name: Install python-requests module
+- name: Install urllib3 with pip
+ pip:
+ name: urllib3
+ version: 1.10.2
+
+- name: Install requests with pip
pip:
name: requests
version: 2.6.1
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml
similarity index 88%
copy from metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
copy to metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml
index 52d6ca1..72a96d4 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-CentOS-7.yml
@@ -15,7 +15,12 @@
# limitations under the License.
#
---
-- name: Install python-requests module
+- name: Install urllib3 with pip
+ pip:
+ name: urllib3
+ version: 1.10.2
+
+- name: Install requests with pip
pip:
name: requests
version: 2.6.1
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-Ubuntu-14.yml
similarity index 90%
rename from metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
rename to metron-deployment/ansible/roles/ambari_config/tasks/dependencies-Ubuntu-14.yml
index 52d6ca1..964acf2 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/dependencies.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/dependencies-Ubuntu-14.yml
@@ -15,7 +15,4 @@
# limitations under the License.
#
---
-- name: Install python-requests module
- pip:
- name: requests
- version: 2.6.1
+# nothing to do
diff --git a/metron-deployment/ansible/roles/ambari_config/tasks/main.yml b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml
index d5f38b1..d5ab637 100644
--- a/metron-deployment/ansible/roles/ambari_config/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_config/tasks/main.yml
@@ -17,7 +17,7 @@
---
- include_vars: "{{ cluster_type }}.yml"
-- include: dependencies.yml
+- include_tasks: "dependencies-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml"
- name : Wait for Ambari to start; http://{{ ambari_host }}:{{ ambari_port }}
wait_for :
diff --git a/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml b/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml
index 5351a60..94bd95a 100644
--- a/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml
+++ b/metron-deployment/ansible/roles/ambari_gather_facts/defaults/main.yml
@@ -17,3 +17,7 @@
---
curl: "curl -s -u {{ ambari_user }}:{{ ambari_password }} -X GET -H \"X-Requested-By: ambari\""
parse_json: "import sys, json; print json.load(sys.stdin)"
+ambari_host: "{{ groups.ambari_master[0] }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/hostname.yml
similarity index 88%
copy from metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
copy to metron-deployment/ansible/roles/ambari_slave/tasks/hostname.yml
index b78a2a7..4cfd115 100644
--- a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/hostname.yml
@@ -15,8 +15,6 @@
# limitations under the License.
#
---
-- include: install-agent.yml
-
- name: Create ambari-agent hostname script
template:
src: "../roles/ambari_slave/files/hostname.sh"
@@ -35,12 +33,3 @@
with_items:
- { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
- { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
-
-- name: Ensure ambari-agent is running
- service:
- name: ambari-agent
- state: restarted
- enabled: yes
-
-- name : Wait for agent to register
- command : sleep 10
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-6.yml
similarity index 92%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
copy to metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-6.yml
index 9fdda7e..8f16b92 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-6.yml
@@ -15,5 +15,5 @@
# limitations under the License.
#
---
-- name: Disable firewall on Ubuntu
- shell: ufw disable
+- name: Install ambari-agent on CentOS 6
+ yum: name=ambari-agent
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-7.yml
similarity index 76%
copy from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
copy to metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-7.yml
index 62d0027..d64f21f 100644
--- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-CentOS-7.yml
@@ -15,10 +15,12 @@
# limitations under the License.
#
---
-- name: Install ambari-agent on CentOS
+- name: Install ambari-agent on CentOS 7
yum: name=ambari-agent
- when: ansible_distribution == "CentOS"
-- name: Install ambari-agent on Ubuntu
- apt: name=ambari-agent force=yes
- when: ansible_distribution == "Ubuntu"
+- name: Force Ambari Agent to use TLSv1.2
+ lineinfile:
+ dest: /etc/ambari-agent/conf/ambari-agent.ini
+ state: present
+ insertafter: '\[security\]'
+ line: "force_https_protocol=PROTOCOL_TLSv1_2"
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-Ubuntu-14.yml
similarity index 91%
rename from metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
rename to metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-Ubuntu-14.yml
index 9fdda7e..ae48214 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-ubuntu.yml
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent-Ubuntu-14.yml
@@ -15,5 +15,5 @@
# limitations under the License.
#
---
-- name: Disable firewall on Ubuntu
- shell: ufw disable
+- name: Install ambari-agent on Ubuntu 14
+ apt: name=ambari-agent force=yes
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
index b78a2a7..7d9bac0 100644
--- a/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
+++ b/metron-deployment/ansible/roles/ambari_slave/tasks/main.yml
@@ -15,26 +15,9 @@
# limitations under the License.
#
---
-- include: install-agent.yml
+- include_tasks: install-agent-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml
-- name: Create ambari-agent hostname script
- template:
- src: "../roles/ambari_slave/files/hostname.sh"
- dest: "/var/lib/ambari-agent/hostname.sh"
- mode: 0744
- owner: "{{ ambari_installation_user }}"
- group: "{{ ambari_installation_user }}"
-
-- name: Configure ambari-server hostname in ambari-agent configuration
- lineinfile:
- dest: /etc/ambari-agent/conf/ambari-agent.ini
- regexp: "{{ item.regexp }}"
- line: "{{ item.line }}"
- insertafter: "{{ item.insertafter }}"
- backup: yes
- with_items:
- - { regexp: "^.*hostname=.*$", line: "hostname={{ groups.ambari_master[0] }}", insertafter: '\[server\]' }
- - { regexp: "^hostname_script=.*$", line: "hostname_script=/var/lib/ambari-agent/hostname.sh", insertafter: '\[agent\]'}
+- include_tasks: hostname.yml
- name: Ensure ambari-agent is running
service:
diff --git a/metron-deployment/ansible/roles/enable-swap/defaults/main.yml b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml
index 7190606..94a63c2 100644
--- a/metron-deployment/ansible/roles/enable-swap/defaults/main.yml
+++ b/metron-deployment/ansible/roles/enable-swap/defaults/main.yml
@@ -15,5 +15,5 @@
# limitations under the License.
#
---
-swapfile: /swapfile
-swapspace: 4G
+swapfile: /swappy
+swapspace: 8092
diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/check-swap.yml
similarity index 79%
copy from metron-deployment/ansible/roles/enable-swap/tasks/main.yml
copy to metron-deployment/ansible/roles/enable-swap/tasks/check-swap.yml
index 4b159b4..3bd0146 100644
--- a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
+++ b/metron-deployment/ansible/roles/enable-swap/tasks/check-swap.yml
@@ -14,17 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- name: "Is swap space enabled?"
- shell: swapon -s | grep {{ swapfile }}
+---
+- shell: swapon -s | grep {{ swapfile }}
register: swapcheck
failed_when: swapcheck.rc != 0 and swapcheck.rc != 1
-- debug: msg="Swap space is already enabled"
+- shell: free -m | grep Swap | awk -F" " '{ print $2}'
+ register: actual_swapspace
+
+- debug: msg="Swap is enabled with {{ actual_swapspace.stdout }} mb"
when: swapcheck.rc == 0
-- debug: msg="Swap space is NOT yet enabled"
+- debug: msg="Swap is NOT enabled"
when: swapcheck.rc == 1
-
-- include_tasks: enable-swap.yml
- when: swapcheck.rc != 0
diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
index 4b159b4..8247ab1 100644
--- a/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
+++ b/metron-deployment/ansible/roles/enable-swap/tasks/main.yml
@@ -14,17 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- name: "Is swap space enabled?"
- shell: swapon -s | grep {{ swapfile }}
- register: swapcheck
- failed_when: swapcheck.rc != 0 and swapcheck.rc != 1
+---
+- include_tasks: check-swap.yml
-- debug: msg="Swap space is already enabled"
- when: swapcheck.rc == 0
+#
+# only enable swap if the swapfile does not already exist
+#
+- name: "Checking for {{ swapfile }}"
+ stat: path="{{ swapfile }}"
+ register: swap
-- debug: msg="Swap space is NOT yet enabled"
- when: swapcheck.rc == 1
+- include_tasks: setup-swap.yml
+ when: swap.stat.exists == False
-- include_tasks: enable-swap.yml
- when: swapcheck.rc != 0
+- debug: msg="No swap changes necessary"
+ when: swap.stat.exists == True
diff --git a/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml b/metron-deployment/ansible/roles/enable-swap/tasks/setup-swap.yml
similarity index 84%
rename from metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml
rename to metron-deployment/ansible/roles/enable-swap/tasks/setup-swap.yml
index fc6c094..648c208 100644
--- a/metron-deployment/ansible/roles/enable-swap/tasks/enable-swap.yml
+++ b/metron-deployment/ansible/roles/enable-swap/tasks/setup-swap.yml
@@ -15,8 +15,11 @@
# limitations under the License.
#
---
-- name: "Allocate {{ swapspace }} for swap space"
- shell: fallocate -l {{ swapspace }} {{ swapfile }}
+- name: "Disable swap space"
+ shell: swapoff -a
+
+- name: "Allocate {{ swapspace }} mb for swap space"
+ shell: dd if=/dev/zero of={{ swapfile }} count={{ swapspace }} bs=1MiB
- name: "Permissioning {{ swapfile }}"
file:
@@ -33,3 +36,5 @@
lineinfile:
dest: "/etc/fstab"
line: "{{ swapfile }} none swap sw 0 0"
+
+- include_tasks: check-swap.yml
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-6.yml
similarity index 82%
copy from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
copy to metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-6.yml
index 27e67d5..022e49f 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-6.yml
@@ -15,6 +15,8 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
+- name: Install init.d service
+ template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755
+
+- name: Start sensor stubs
+ service: name=sensor-stubs state=restarted
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-7.yml
similarity index 68%
copy from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
copy to metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-7.yml
index 62d0027..76293ad 100644
--- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
+++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-CentOS-7.yml
@@ -15,10 +15,18 @@
# limitations under the License.
#
---
-- name: Install ambari-agent on CentOS
- yum: name=ambari-agent
- when: ansible_distribution == "CentOS"
+- name: Install systemd services
+ template: src={{ item }} dest=/etc/systemd/system/ mode=0755
+ with_items:
+ - sensor-stubs-bro.service
+ - sensor-stubs-yaf.service
+ - sensor-stubs-snort.service
-- name: Install ambari-agent on Ubuntu
- apt: name=ambari-agent force=yes
- when: ansible_distribution == "Ubuntu"
+- name: Start sensor stubs
+ systemd:
+ name: "{{ item }}"
+ state: restarted
+ with_items:
+ - sensor-stubs-bro
+ - sensor-stubs-yaf
+ - sensor-stubs-snort
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-Ubuntu-14.yml
similarity index 82%
rename from metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
rename to metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-Ubuntu-14.yml
index 27e67d5..022e49f 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/iptables-centos.yml
+++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/install-service-Ubuntu-14.yml
@@ -15,6 +15,8 @@
# limitations under the License.
#
---
-- name: Stop iptables on CentOS
- ignore_errors: yes
- service: name=iptables state=stopped enabled=no
+- name: Install init.d service
+ template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755
+
+- name: Start sensor stubs
+ service: name=sensor-stubs state=restarted
diff --git a/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml b/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml
index c333025..2e05d99 100644
--- a/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml
+++ b/metron-deployment/ansible/roles/sensor-stubs/tasks/main.yml
@@ -29,15 +29,11 @@
- snort.out
- yaf.out
-- name: Install service script
- template: src=sensor-stubs dest=/etc/init.d/sensor-stubs mode=0755
-
-- name: Install sensor stubs
+- name: Install sensor stubs scripts
template: src={{ item }} dest={{ sensor_stubs_bin }}/ mode=0755
with_items:
- start-bro-stub
- start-snort-stub
- start-yaf-stub
-- name: Start sensor stubs
- service: name=sensor-stubs state=restarted
+- include_tasks: install-service-{{ansible_distribution}}-{{ansible_distribution_major_version}}.yml
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-bro.service b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-bro.service
new file mode 100644
index 0000000..79094ae
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-bro.service
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[Unit]
+ Description=Sends canned Bro telemetry to a Kafka topic
+
+[Service]
+ ExecStart={{ sensor_stubs_bin }}/start-bro-stub
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-snort.service b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-snort.service
new file mode 100644
index 0000000..5f4ec25
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-snort.service
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[Unit]
+ Description=Sends canned Snort telemetry to a Kafka topic
+
+[Service]
+ ExecStart={{ sensor_stubs_bin }}/start-snort-stub
diff --git a/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-yaf.service b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-yaf.service
new file mode 100644
index 0000000..aa4bebb
--- /dev/null
+++ b/metron-deployment/ansible/roles/sensor-stubs/templates/sensor-stubs-yaf.service
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[Unit]
+ Description=Sends canned YAF telemetry to a Kafka topic
+
+[Service]
+ ExecStart={{ sensor_stubs_bin }}/start-yaf-stub
diff --git a/metron-deployment/development/centos7/Vagrantfile b/metron-deployment/development/centos7/Vagrantfile
new file mode 100644
index 0000000..4706e42
--- /dev/null
+++ b/metron-deployment/development/centos7/Vagrantfile
@@ -0,0 +1,108 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+require 'getoptlong'
+
+ansibleTags=''
+ansibleSkipTags='sensors'
+
+begin
+ opts = GetoptLong.new(
+ [ '--ansible-tags', GetoptLong::OPTIONAL_ARGUMENT ],
+ [ '--ansible-skip-tags', GetoptLong::OPTIONAL_ARGUMENT ]
+ )
+
+ opts.quiet = true
+
+ opts.each do |opt, arg|
+ case opt
+ when '--ansible-tags'
+ ansibleTags=arg
+ when '--ansible-skip-tags'
+ ansibleSkipTags=arg
+ end
+ end
+rescue Exception => ignored
+#Ignore to allow other opts to be passed to Vagrant
+end
+
+puts " Running with ansible-tags: " + ansibleTags.split(",").to_s if ansibleTags != ''
+puts " Running with ansible-skip-tags: " + ansibleSkipTags.split(",").to_s if ansibleSkipTags != ''
+
+hosts = [{
+ hostname: "node1",
+ ip: "192.168.66.121",
+ memory: "8192",
+ cpus: 4,
+ promisc: 2 # enables promisc on the 'Nth' network interface
+}]
+
+Vagrant.configure(2) do |config|
+
+ # host built on centos
+ config.vm.box = "centos/7"
+ config.ssh.insert_key = true
+
+ # enable the hostmanager plugin
+ config.hostmanager.enabled = true
+ config.hostmanager.manage_host = true
+
+ # enable vagrant cachier if present
+ if Vagrant.has_plugin?("vagrant-cachier")
+ config.cache.enable :yum
+ config.cache.scope = :box
+
+ config.cache.synced_folder_opts = {
+ type: :nfs,
+ mount_options: ['rw', 'vers=3', 'tcp', 'nolock']
+ }
+ end
+
+ # host definition
+ hosts.each_with_index do |host, index|
+ config.vm.define host[:hostname] do |node|
+
+ # host settings
+ node.vm.hostname = host[:hostname]
+ node.vm.network "private_network", ip: host[:ip]
+
+ # vm settings
+ node.vm.provider "virtualbox" do |vb|
+ vb.memory = host[:memory]
+ vb.cpus = host[:cpus]
+
+ # enable promisc mode on the network interface
+ if host.has_key?(:promisc)
+ vb.customize ["modifyvm", :id, "--nicpromisc#{host[:promisc]}", "allow-all"]
+ end
+ # disable audio, so that the vm doesn't capture the sound / mic
+ vb.customize ["modifyvm", :id, "--audio", "none"]
+ end
+ end
+ end
+
+ # provision the host with ansible
+ config.vm.provision :ansible do |ansible|
+ ansible.playbook = "ansible/playbook.yml"
+ ansible.become = true
+ ansible.tags = ansibleTags.split(",") if ansibleTags != ''
+ ansible.skip_tags = ansibleSkipTags.split(",") if ansibleSkipTags != ''
+ ansible.inventory_path = "ansible/inventory"
+ ansible.compatibility_mode = "auto"
+ ansible.raw_arguments = [
+ ]
+ end
+end
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/development/centos7/ansible.cfg
similarity index 71%
copy from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
copy to metron-deployment/development/centos7/ansible.cfg
index 62d0027..66f0afd 100644
--- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
+++ b/metron-deployment/development/centos7/ansible.cfg
@@ -14,11 +14,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- name: Install ambari-agent on CentOS
- yum: name=ambari-agent
- when: ansible_distribution == "CentOS"
+[defaults]
+host_key_checking = false
+library = ../../ansible/extra_modules
+roles_path = ../../ansible/roles
+pipelining = True
+log_path = ./ansible.log
+callback_plugins = ../../ansible/callback_plugins
-- name: Install ambari-agent on Ubuntu
- apt: name=ambari-agent force=yes
- when: ansible_distribution == "Ubuntu"
+# fix for "ssh throws 'unix domain socket too long' " problem
+[ssh_connection]
+control_path = %(directory)s/%%h-%%p-%%r
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml b/metron-deployment/development/centos7/ansible/inventory/group_vars/all
similarity index 71%
rename from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
rename to metron-deployment/development/centos7/ansible/inventory/group_vars/all
index 23c4aca..ba37eed 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-ubuntu.yml
+++ b/metron-deployment/development/centos7/ansible/inventory/group_vars/all
@@ -14,11 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- name: Setup Ambari repo on Ubuntu
- shell: echo "deb {{ ubuntu_ambari_repo }} Ambari main" | tee /etc/apt/sources.list.d/ambari.list
- become: true
+# only need to build the RPMs for CentOS
+metron_build_packages_cmd: "shell cd {{ metron_build_dir }}/metron-deployment && mvn clean package -DskipTests -Pbuild-rpms"
-- name: Update package cache on Ubuntu
- apt: update_cache=yes
+cluster_type: single_node_vm
+elasticsearch_hosts: "{{ groups.search | join(',') }}"
+hdp_host_group: "{{ groups.ambari_slave }}"
+hdp_stack: "2.6"
diff --git a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml b/metron-deployment/development/centos7/ansible/inventory/hosts
similarity index 79%
rename from metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
rename to metron-deployment/development/centos7/ansible/inventory/hosts
index 8c1bc33..9bd9ea1 100644
--- a/metron-deployment/ansible/roles/ambari_common/tasks/ambari-repo-centos.yml
+++ b/metron-deployment/development/centos7/ansible/inventory/hosts
@@ -14,9 +14,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
----
-- name: Setup Ambari repo on CentOS
- get_url: url="{{ centos_ambari_install_url }}" dest=/etc/yum.repos.d/ambari.repo
-- name: Update package cache on CentOS
- yum: name=* update_cache=yes
+[ambari_master]
+node1
+
+[ambari_slave]
+node1
+
+[metron]
+node1
+
+[search]
+node1
+
+[sensors]
+node1
+
+[pcap_server]
+node1
+
+[web]
+node1
+
+[zeppelin]
+node1
+
+[monit:children]
+sensors
+pcap_server
+
+[local]
+127.0.0.1
diff --git a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml b/metron-deployment/development/centos7/ansible/playbook.yml
similarity index 75%
rename from metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
rename to metron-deployment/development/centos7/ansible/playbook.yml
index 62d0027..433d3b5 100644
--- a/metron-deployment/ansible/roles/ambari_slave/tasks/install-agent.yml
+++ b/metron-deployment/development/centos7/ansible/playbook.yml
@@ -15,10 +15,10 @@
# limitations under the License.
#
---
-- name: Install ambari-agent on CentOS
- yum: name=ambari-agent
- when: ansible_distribution == "CentOS"
+- hosts: all
+ roles:
+ - { role: libselinux-python, tags: libselinux-python }
+ - { role: enable-swap, tags: enable-swap }
+ - { role: enable-remote-ssh, tags: enable-remote-ssh }
-- name: Install ambari-agent on Ubuntu
- apt: name=ambari-agent force=yes
- when: ansible_distribution == "Ubuntu"
+- import_playbook: ../../../ansible/playbooks/metron_full_install.yml