You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metron.apache.org by ce...@apache.org on 2016/03/21 18:06:11 UTC
[29/43] incubator-metron git commit: METRON-59 Automated Deployment
of Metron to Amazon EC2 (nickwallen via cestella) closes
apache/incubator-metron#41
METRON-59 Automated Deployment of Metron to Amazon EC2 (nickwallen via cestella) closes apache/incubator-metron#41
Project: http://git-wip-us.apache.org/repos/asf/incubator-metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metron/commit/3397fb22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metron/tree/3397fb22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metron/diff/3397fb22
Branch: refs/heads/Metron_0.1BETA
Commit: 3397fb221630f4f60395a34dfd173bf789b5d279
Parents: d043f24
Author: nickwallen <ni...@nickallen.org>
Authored: Wed Mar 9 14:52:29 2016 -0500
Committer: cstella <ce...@gmail.com>
Committed: Wed Mar 9 14:52:29 2016 -0500
----------------------------------------------------------------------
deployment/amazon-ec2/.gitignore | 2 +
deployment/amazon-ec2/README.md | 44 ++++++++
deployment/amazon-ec2/ansible.cfg | 27 +++++
deployment/amazon-ec2/conf/defaults.yml | 77 ++++++++++++++
deployment/amazon-ec2/conf/ec2.ini | 105 +++++++++++++++++++
deployment/amazon-ec2/playbook.yml | 49 +++++++++
deployment/amazon-ec2/tasks/create-hosts.yml | 58 ++++++++++
deployment/amazon-ec2/tasks/create-keypair.yml | 23 ++++
.../create-open-inbound-security-group.yml | 26 +++++
.../create-open-outbound-security-group.yml | 26 +++++
.../amazon-ec2/tasks/create-security-group.yml | 28 +++++
deployment/amazon-ec2/tasks/create-vpc.yml | 46 ++++++++
deployment/amazon-ec2/tasks/expand-volume.yml | 27 +++++
deployment/amazon-ec2/tasks/mount-volume.yml | 25 +++++
deployment/playbooks/ambari_install.yml | 12 ++-
deployment/playbooks/metron_install.yml | 23 ++--
16 files changed, 586 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/.gitignore
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/.gitignore b/deployment/amazon-ec2/.gitignore
new file mode 100644
index 0000000..e066c7b
--- /dev/null
+++ b/deployment/amazon-ec2/.gitignore
@@ -0,0 +1,2 @@
+*.pem
+*.secret
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/README.md
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/README.md b/deployment/amazon-ec2/README.md
new file mode 100644
index 0000000..bd0a50b
--- /dev/null
+++ b/deployment/amazon-ec2/README.md
@@ -0,0 +1,44 @@
+Apache Metron on Amazon EC2
+===========================
+
+This project fully automates the provisioning of Apache Metron on Amazon EC2 infrastructure. Starting with only your Amazon EC2 credentials, this project will create a fully-functioning, end-to-end, multi-node cluster running Apache Metron.
+
+Getting Started
+---------------
+
+The host that will be used to drive the provisioning process will need to have [Ansible](https://github.com/ansible/ansible), Python and PIP installed. In most cases, a development laptop serves this purpose just fine. For better performance, run this playbook on a pre-provisioned EC2 host.
+
+Ensure that an SSH key has been generated. The playbook will attempt to register `~/.ssh/id_rsa.pub` with each provisioned host so that an SSH connection can be established. If one does not exist, use `ssh-keygen` to create one.
+
+1. Install the Python library `boto` and its dependencies. This enables Ansible to communicate with Amazon EC2.
+
+ ```
+ pip install boto six
+ ```
+
+2. Use Amazon's [IAM](https://console.aws.amazon.com/iam/) tool to generate an access key. Export these access keys in an environment variable so that Ansible can authenticate with Amazon EC2.
+
+ ```
+ export AWS_ACCESS_KEY_ID="..."
+ export AWS_SECRET_ACCESS_KEY="..."
+ ```
+
+3. Build Metron's streaming topology uber-jar.
+
+ ```
+ cd ../../metron-streaming
+ mvn clean package -DskipTests
+ ```
+
+4. Kick-off the provisioning playbook. If the playbook fails mid-stream for any reason, simply re-run it. This will attempt to re-provision on the previously instantiated EC2 hosts.
+
+ ```
+ export EC2_INI_PATH=conf/ec2.ini
+ ansible-playbook -i ec2.py playbook.yml
+ ```
+
+Each of the provisioned hosts will be externally accessible from the internet at-large. Connecting to one over SSH as the user `centos` will not require a password as it will authenticate with the pre-defined SSH key.
+
+```
+ssh centos@ec2-52-91-215-174.compute-1.amazonaws.com
+```
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/ansible.cfg
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/ansible.cfg b/deployment/amazon-ec2/ansible.cfg
new file mode 100644
index 0000000..e79770c
--- /dev/null
+++ b/deployment/amazon-ec2/ansible.cfg
@@ -0,0 +1,27 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[defaults]
+host_key_checking = False
+library = ../extra_modules
+roles_path = ../roles
+pipelining = True
+remote_user = centos
+forks = 20
+
+# fix for "ssh throws 'unix domain socket too long' " problem
+[ssh_connection]
+control_path = %(directory)s/%%h-%%p-%%r
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/conf/defaults.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/conf/defaults.yml b/deployment/amazon-ec2/conf/defaults.yml
new file mode 100644
index 0000000..0c260b9
--- /dev/null
+++ b/deployment/amazon-ec2/conf/defaults.yml
@@ -0,0 +1,77 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+# ec2
+region: us-west-2
+instance_type: m4.xlarge
+image: ami-05cf2265
+volume_type: standard
+key_name: metron-test
+env: metron-test
+
+# ambari
+ambari_host: "{{ groups.ambari_master[0] }}"
+ambari_port: 8080
+ambari_user: admin
+ambari_password: admin
+cluster_type: small_cluster
+
+# hbase
+pcap_hbase_table: pcap
+tracker_hbase_table: access_tracker
+threatintel_ip_hbase_table: malicious_ip
+
+# kafka
+pycapa_topic: pcap
+bro_topic: bro
+yaf_topic: ipfix
+snort_topic: snort
+num_partitions: 3
+retention_in_gb: 25
+
+# elasticsearch
+elasticsearch_transport_port: 9300
+elasticsearch_network_interface: eth0
+elasticsearch_web_port: 9200
+
+# metron variables
+metron_version: 0.1BETA
+java_home: /usr/jdk64/jdk1.8.0_40
+pcapservice_port: 8081
+sniff_interface: tap0
+snort_alert_csv_path: "/var/log/snort/alert.csv"
+yaf_args: "--idle-timeout 0"
+
+# pcap-replay
+pcap_replay: True
+pcap_replay_interface: tap0
+install_tap: True
+
+# data directories
+zookeeper_data_dir: "/data1/hadoop/zookeeper"
+namenode_checkpoint_dir: "/data1/hadoop/hdfs/namesecondary"
+namenode_name_dir: "/data1/hadoop/hdfs/namenode"
+datanode_data_dir: "/data1/hadoop/hdfs/data,/data2/hadoop/hdfs/data"
+journalnode_edits_dir: "/data1/hadoop/hdfs/journalnode"
+nodemanager_local_dirs: "/data1/hadoop/yarn/local"
+timeline_ldb_store_path: "/data1/hadoop/yarn/timeline"
+timeline_ldb_state_path: "/data1/hadoop/yarn/timeline"
+nodemanager_log_dirs: "/data1/hadoop/yarn/log"
+jhs_recovery_store_ldb_path: "/data1/hadoop/mapreduce/jhs"
+storm_local_dir: "/data1/hadoop/storm"
+kafka_log_dirs: "/data2/kafka-log"
+elasticsearch_data_dir: "/data1/elasticsearch,/data2/elasticsearch"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/conf/ec2.ini
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/conf/ec2.ini b/deployment/amazon-ec2/conf/ec2.ini
new file mode 100755
index 0000000..646ffaf
--- /dev/null
+++ b/deployment/amazon-ec2/conf/ec2.ini
@@ -0,0 +1,105 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Ansible EC2 external inventory script settings.
+#
+# Refer to https://github.com/ansible/ansible/blob/devel/contrib/inventory/ec2.ini
+# for additional information on available settings
+#
+
+[ec2]
+
+# AWS regions to make calls to. Set this to 'all' to make request to all regions
+# in AWS and merge the results together. Alternatively, set this to a comma
+# separated list of regions. E.g. 'us-east-1,us-west-1,us-west-2'
+regions = all
+
+# When generating inventory, Ansible needs to know how to address a server.
+# Each EC2 instance has a lot of variables associated with it. Here is the list:
+# http://docs.pythonboto.org/en/latest/ref/ec2.html#module-boto.ec2.instance
+# Below are 2 variables that are used as the address of a server:
+# - destination_variable
+# - vpc_destination_variable
+
+# This is the normal destination variable to use. If you are running Ansible
+# from outside EC2, then 'public_dns_name' makes the most sense. If you are
+# running Ansible from within EC2, then perhaps you want to use the internal
+# address, and should set this to 'private_dns_name'. The key of an EC2 tag
+# may optionally be used; however the boto instance variables hold precedence
+# in the event of a collision.
+destination_variable = public_dns_name
+
+# For server inside a VPC, using DNS names may not make sense. When an instance
+# has 'subnet_id' set, this variable is used. If the subnet is public, setting
+# this to 'ip_address' will return the public IP address. For instances in a
+# private subnet, this should be set to 'private_ip_address', and Ansible must
+# be run from within EC2. The key of an EC2 tag may optionally be used; however
+# the boto instance variables hold precedence in the event of a collision.
+# WARNING: - instances that are in the private vpc, _without_ public ip address
+# will not be listed in the inventory until You set:
+# vpc_destination_variable = private_ip_address
+#vpc_destination_variable = ip_address
+vpc_destination_variable = public_dns_name
+
+# To exclude RDS instances from the inventory, uncomment and set to False.
+#rds = False
+
+# To exclude ElastiCache instances from the inventory, uncomment and set to False.
+#elasticache = False
+
+# API calls to EC2 are slow. For this reason, we cache the results of an API
+# call. Set this to the path you want cache files to be written to. Two files
+# will be written to this directory:
+# - ansible-ec2.cache
+# - ansible-ec2.index
+cache_path = ~/.ansible/tmp
+
+# The number of seconds a cache file is considered valid. After this many
+# seconds, a new API call will be made, and the cache file will be updated.
+# To disable the cache, set this value to 0
+cache_max_age = 300
+
+# Organize groups into a nested/hierarchy instead of a flat namespace.
+nested_groups = False
+
+# Replace - tags when creating groups to avoid issues with ansible
+replace_dash_in_groups = True
+
+# If set to true, any tag of the form "a,b,c" is expanded into a list
+# and the results are used to create additional tag_* inventory groups.
+expand_csv_tags = False
+
+# The EC2 inventory output can become very large. To manage its size,
+# configure which groups should be created.
+group_by_instance_id = True
+group_by_region = True
+group_by_availability_zone = True
+group_by_ami_id = True
+group_by_instance_type = True
+group_by_key_pair = True
+group_by_vpc_id = True
+group_by_security_group = True
+group_by_tag_keys = True
+group_by_tag_none = True
+group_by_route53_names = True
+group_by_rds_engine = True
+group_by_rds_parameter_group = True
+group_by_elasticache_engine = True
+group_by_elasticache_cluster = True
+group_by_elasticache_parameter_group = True
+group_by_elasticache_replication_group = True
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/playbook.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/playbook.yml b/deployment/amazon-ec2/playbook.yml
new file mode 100644
index 0000000..b269cfa
--- /dev/null
+++ b/deployment/amazon-ec2/playbook.yml
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+#
+# instantiate the hosts on amazon ec2
+#
+- hosts: localhost
+ vars_files:
+ - conf/defaults.yml
+ tasks:
+ - include: tasks/create-keypair.yml
+ - include: tasks/create-vpc.yml
+ - include: tasks/create-open-inbound-security-group.yml
+ - include: tasks/create-open-outbound-security-group.yml
+ - include: tasks/create-hosts.yml host_count=1 host_type=sensors,ambari_master,metron,ec2
+ - include: tasks/create-hosts.yml host_count=5 host_type=ambari_slave,ec2
+ - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,hadoop_client,metron,ec2
+ - include: tasks/create-hosts.yml host_count=1 host_type=ambari_slave,enrichment,metron,ec2
+ - include: tasks/create-hosts.yml host_count=3 host_type=search,metron,ec2
+ - include: tasks/create-hosts.yml host_count=1 host_type=web,mysql,metron,ec2
+
+#
+# mount additional data volumes on all ec2 hosts
+#
+- hosts: ec2
+ become: True
+ tasks:
+ - include: tasks/mount-volume.yml vol_src=/dev/xvdb vol_mnt=/data1
+ - include: tasks/mount-volume.yml vol_src=/dev/xvdc vol_mnt=/data2
+ - include: tasks/expand-volume.yml volume=/dev/xvda
+
+#
+# build the metron cluster
+#
+- include: ../playbooks/metron_full_install.yml
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/create-hosts.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-hosts.yml b/deployment/amazon-ec2/tasks/create-hosts.yml
new file mode 100644
index 0000000..f162d7f
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/create-hosts.yml
@@ -0,0 +1,58 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Instantiate '{{ host_count }}' host(s) as '{{ host_type }}'
+ ec2:
+ region: "{{ region }}"
+ instance_type: "{{ instance_type }}"
+ image: "{{ image }}"
+ key_name: "{{ key_name }}"
+ assign_public_ip: True
+ group: [vpc-all-inbound,vpc-all-outbound]
+ vpc_subnet_id: "{{ vpc.subnets[0].id }}"
+ instance_tags:
+ Name: "[{{ env }}] {{ host_type }}"
+ type: "{{ host_type }}"
+ env: "{{ env }}"
+ exact_count: "{{ host_count }}"
+ count_tag:
+ type: "{{ host_type }}"
+ volumes:
+ - device_name: /dev/sda1
+ volume_type: "{{ volume_type }}"
+ volume_size: 50
+ delete_on_termination: true
+ - device_name: /dev/xvdb
+ volume_type: "{{ volume_type }}"
+ volume_size: 100
+ delete_on_termination: true
+ - device_name: /dev/xvdc
+ volume_type: "{{ volume_type }}"
+ volume_size: 100
+ delete_on_termination: true
+ wait: yes
+ register: ec2
+
+- name: Add host(s) to a hostgroup
+ add_host: hostname={{ item.public_dns_name }} groups={{ host_type }}
+ with_items: ec2.tagged_instances
+ when: item.public_dns_name is defined
+
+- name: Wait for connectivity to host(s)
+ wait_for: host={{ item.public_dns_name }} port=22 search_regex=OpenSSH
+ with_items: ec2.tagged_instances
+ when: item.public_dns_name is defined
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/create-keypair.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-keypair.yml b/deployment/amazon-ec2/tasks/create-keypair.yml
new file mode 100644
index 0000000..b26a6e4
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/create-keypair.yml
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Define keypair
+ ec2_key:
+ name: "{{ key_name }}"
+ region: "{{ region }}"
+ key_material: "{{ item }}"
+ with_file: "{{ key_file | default('~/.ssh/id_rsa.pub') }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/create-open-inbound-security-group.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-open-inbound-security-group.yml b/deployment/amazon-ec2/tasks/create-open-inbound-security-group.yml
new file mode 100644
index 0000000..9c6e5da
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/create-open-inbound-security-group.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: WARNING Define open inbound security group
+ ec2_group:
+ name: vpc-all-inbound
+ description: WARNING allow all inbound connections from the internet
+ region: "{{ region }}"
+ vpc_id: "{{ vpc_id }}"
+ rules:
+ - proto: all
+ cidr_ip: 0.0.0.0/0
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/create-open-outbound-security-group.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-open-outbound-security-group.yml b/deployment/amazon-ec2/tasks/create-open-outbound-security-group.yml
new file mode 100644
index 0000000..3d53026
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/create-open-outbound-security-group.yml
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Define outbound security group
+ ec2_group:
+ name: vpc-all-outbound
+ description: allow all outbound connections to the internet
+ region: "{{ region }}"
+ vpc_id: "{{ vpc_id }}"
+ rules_egress:
+ - proto: all
+ cidr_ip: 0.0.0.0/0
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/create-security-group.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-security-group.yml b/deployment/amazon-ec2/tasks/create-security-group.yml
new file mode 100644
index 0000000..c52271d
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/create-security-group.yml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Define "{{ name }}" security group
+ ec2_group:
+ name: "{{ name }}"
+ region: "{{ region }}"
+ description: "{{ name }} - {{ proto }} - {{ port }}"
+ vpc_id: "{{ vpc_id }}"
+ rules:
+ - proto: "{{ proto }}"
+ from_port: "{{ port }}"
+ to_port: "{{ port }}"
+ cidr_ip: 0.0.0.0/0
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/create-vpc.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/create-vpc.yml b/deployment/amazon-ec2/tasks/create-vpc.yml
new file mode 100644
index 0000000..7af14a1
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/create-vpc.yml
@@ -0,0 +1,46 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+ - name: Create vpc
+ ec2_vpc:
+ region: "{{ region }}"
+ internet_gateway: True
+ resource_tags: { Name: Metron }
+ cidr_block: 10.0.0.0/16
+ subnets:
+ - cidr: 10.0.0.0/24
+ resource_tags:
+ tier: web
+ - cidr: 10.0.1.0/24
+ resource_tags:
+ tier: hdp
+ - cidr: 10.0.2.0/24
+ resource_tags:
+ tier: sensors
+ route_tables:
+ - subnets:
+ - 10.0.0.0/24
+ - 10.0.1.0/24
+ - 10.0.2.0/24
+ routes:
+ - dest: 0.0.0.0/0
+ gw: igw
+ register: vpc
+
+ - name: Created vpc with id='"{{ vpc.vpc_id }}"'
+ set_fact:
+ vpc_id: "{{ vpc.vpc_id }}"
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/expand-volume.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/expand-volume.yml b/deployment/amazon-ec2/tasks/expand-volume.yml
new file mode 100644
index 0000000..2a80ef5
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/expand-volume.yml
@@ -0,0 +1,27 @@
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Expand {{ volume }} volume
+# sectors delete 1 new primary first past mbr to end bootable write and exit
+ shell: "(echo u s; echo d 1; echo n; echo p; echo 1; echo 2048 ; echo ;echo a; echo 1; echo w) | fdisk {{ volume }} || true"
+ args:
+ executable: /bin/bash
+
+- name: Restart host(s)
+ command: shutdown -r now "Trigger volume changes"
+ async: 0
+ poll: 0
+ ignore_errors: True
+
+- name: Wait for connectivity to host(s)
+ local_action: wait_for host={{ inventory_hostname }} state=started
+ become: False
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/amazon-ec2/tasks/mount-volume.yml
----------------------------------------------------------------------
diff --git a/deployment/amazon-ec2/tasks/mount-volume.yml b/deployment/amazon-ec2/tasks/mount-volume.yml
new file mode 100644
index 0000000..9609816
--- /dev/null
+++ b/deployment/amazon-ec2/tasks/mount-volume.yml
@@ -0,0 +1,25 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+---
+- name: Install xfsprogs
+ yum: name=xfsprogs
+
+- name: Format data volume(s)
+ filesystem: fstype=xfs dev={{ vol_src }}
+
+- name: Mount the volume
+ mount: name={{ vol_mnt }} src={{ vol_src }} opts=noatime fstype=xfs state=mounted
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/playbooks/ambari_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/ambari_install.yml b/deployment/playbooks/ambari_install.yml
index c7f8249..68d067e 100644
--- a/deployment/playbooks/ambari_install.yml
+++ b/deployment/playbooks/ambari_install.yml
@@ -15,8 +15,14 @@
# limitations under the License.
#
---
+- hosts: ec2
+ become: true
+ tasks:
+ - debug: msg="Detected EC2 - including defaults"
+ - include_vars: ../amazon-ec2/conf/defaults.yml
+
- hosts: ambari_*
- sudo: yes
+ become: true
roles:
- role: ambari_common
tags:
@@ -24,7 +30,7 @@
- hdp-install
- hosts: ambari_master
- sudo: yes
+ become: true
roles:
- role: ambari_master
tags:
@@ -32,7 +38,7 @@
- hdp-install
- hosts: ambari_slave
- sudo: yes
+ become: true
roles:
- role: ambari_slave
tags:
http://git-wip-us.apache.org/repos/asf/incubator-metron/blob/3397fb22/deployment/playbooks/metron_install.yml
----------------------------------------------------------------------
diff --git a/deployment/playbooks/metron_install.yml b/deployment/playbooks/metron_install.yml
index 46565cf..daa512a 100644
--- a/deployment/playbooks/metron_install.yml
+++ b/deployment/playbooks/metron_install.yml
@@ -15,15 +15,21 @@
# limitations under the License.
#
---
+- hosts: ec2
+ become: true
+ tasks:
+ - debug: msg="Detected EC2 - including defaults"
+ - include_vars: ../amazon-ec2/conf/defaults.yml
+
- hosts: metron
- sudo: yes
+ become: true
roles:
- role: metron_common
tags:
- metron-prereqs
- hosts: hadoop_client
- sudo: yes
+ become: true
roles:
- role: ambari_gather_facts
- role: hadoop_setup
@@ -31,7 +37,7 @@
- metron-prereqs
- hosts: search
- sudo: yes
+ become: true
vars:
es_hosts: "{% set comma = joiner(',') %}{% for host in groups['search'] -%}{{ comma() }}{{ host }}{%- endfor %}"
roles:
@@ -40,14 +46,14 @@
- elasticsearch
- hosts: mysql
- sudo: yes
+ become: true
roles:
- role: mysql_server
tags:
- mysql-server
- hosts: ambari_slave
- sudo: yes
+ become: true
roles:
- role: mysql_client
tags:
@@ -55,7 +61,7 @@
- hosts: sensors
- sudo: yes
+ become: true
roles:
- role: ambari_gather_facts
- { role: tap_interface, when: install_tap | default(False) == True }
@@ -69,7 +75,7 @@
- sensors
- hosts: enrichment
- sudo: yes
+ become: true
roles:
- role: ambari_gather_facts
- role: metron_streaming
@@ -77,11 +83,10 @@
- enrichment
- hosts: web
- sudo: yes
+ become: true
roles:
- role: ambari_gather_facts
- role: metron_ui
- role: metron_pcapservice
tags:
- web
-