You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/07/06 17:10:16 UTC
[airavata-data-lake] branch master updated: Ansible initial commit
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 47518c0 Ansible initial commit
47518c0 is described below
commit 47518c0a79a858e48aaebc2f097ac98a9af2a522
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Tue Jul 6 13:09:52 2021 -0400
Ansible initial commit
---
ansible/.gitignore | 1 +
ansible/README.md | 29 ++++++
ansible/datalake.yml | 25 ++++++
ansible/inventories/test/group_vars/all/vars.yml | 32 +++++++
ansible/inventories/test/hosts | 2 +
ansible/requirements.txt | 18 ++++
ansible/roles/data_lake/tasks/main.yml | 31 +++++++
ansible/roles/env_setup/tasks/main.yml | 110 +++++++++++++++++++++++
8 files changed, 248 insertions(+)
diff --git a/ansible/.gitignore b/ansible/.gitignore
new file mode 100644
index 0000000..9549705
--- /dev/null
+++ b/ansible/.gitignore
@@ -0,0 +1 @@
+ENV
\ No newline at end of file
diff --git a/ansible/README.md b/ansible/README.md
new file mode 100644
index 0000000..1de3347
--- /dev/null
+++ b/ansible/README.md
@@ -0,0 +1,29 @@
+# Airavata Datalake Ansible
+
+Ansible script to deploy Apache Airavata and PGA.
+There are ansible roles to install Airavata pre-requisites (RabbitMQ, Zookeeper, MariaDB).
+
+## Ansible installation
+
+Note: the following assumes a Bash shell.
+
+1. Download and install the latest version of Python 3.6. See
+ https://www.python.org/downloads/ or use your system's package manager.
+2. Create a virtual environment in this directory
+
+ cd ansible
+ python3.6 -m venv ENV
+
+3. Source the environment (you'll need to do this each time before using ansible commands)
+
+ source ENV/bin/activate
+
+4. Install ansible and any other dependencies.
+
+ pip install -r requirements.txt
+
+Now you should be ready to run `ansible-playbook` and other ansible commands.
+
+## Useful commands
+
+- Deploy Data Lake: `ansible-playbook -i inventories/path/to/inventory/dir datalake.yml`
\ No newline at end of file
diff --git a/ansible/datalake.yml b/ansible/datalake.yml
new file mode 100644
index 0000000..5651602
--- /dev/null
+++ b/ansible/datalake.yml
@@ -0,0 +1,25 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+- hosts: data-lake
+ roles:
+ - env_setup
+ - data_lake
\ No newline at end of file
diff --git a/ansible/inventories/test/group_vars/all/vars.yml b/ansible/inventories/test/group_vars/all/vars.yml
new file mode 100644
index 0000000..2a9830a
--- /dev/null
+++ b/ansible/inventories/test/group_vars/all/vars.yml
@@ -0,0 +1,32 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+ansible_connection: ssh
+
+user: airavata
+group: airavata
+user_home: "/home/{{ user }}"
+
+# deployment related variables
+deployment_dir: "{{ user_home }}/data-lake-deployment"
+airavata_source_dir: "{{ user_home }}/data-lake-source"
+airavata_repo: "https://github.com/apache/airavata-data-lake.git"
+git_branch: master
diff --git a/ansible/inventories/test/hosts b/ansible/inventories/test/hosts
new file mode 100644
index 0000000..90255bb
--- /dev/null
+++ b/ansible/inventories/test/hosts
@@ -0,0 +1,2 @@
+[data-lake]
+149.165.157.235 ansible_user=ubuntu
\ No newline at end of file
diff --git a/ansible/requirements.txt b/ansible/requirements.txt
new file mode 100644
index 0000000..a1fb8c0
--- /dev/null
+++ b/ansible/requirements.txt
@@ -0,0 +1,18 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ansible~=3.4.0
diff --git a/ansible/roles/data_lake/tasks/main.yml b/ansible/roles/data_lake/tasks/main.yml
new file mode 100644
index 0000000..442324e
--- /dev/null
+++ b/ansible/roles/data_lake/tasks/main.yml
@@ -0,0 +1,31 @@
+- name: open firewall port 8443 for DRMS REST connections
+ firewalld: port="8443/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: open firewall port 7070 for DRMS Grpc connections
+ firewalld: port="7070/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: open firewall port 80 for HTTP connections
+ firewalld: port="80/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: open firewall port 443 for HTTPS connections
+ firewalld: port="443/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: open firewall port 9092 for Kafka connections
+ firewalld: port="9092/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: open firewall port 6060 for Data Orchestrator Grpc connections
+ firewalld: port="6060/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+
diff --git a/ansible/roles/env_setup/tasks/main.yml b/ansible/roles/env_setup/tasks/main.yml
new file mode 100644
index 0000000..85a8715
--- /dev/null
+++ b/ansible/roles/env_setup/tasks/main.yml
@@ -0,0 +1,110 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+#Tasks file can include smaller files if wanted
+#All commons tasks goes here
+- name: Create a new user group "{{ group }}"
+ group: name={{ group }}
+ become: yes
+
+- name: Create a new user "{{ user }}"
+ user: name={{ user }} group={{ group }}
+ become: yes
+
+- name: Install Firewalld (RedHat)
+ yum: name=firewalld state=latest update_cache=yes
+ become: yes
+ when: ansible_os_family == "RedHat"
+
+- name: Install Firewalld (Debian)
+ apt: name=firewalld state=latest update_cache=yes
+ become: yes
+ when: ansible_os_family == "Debian"
+
+
+ # TODO: stop iptables service, can't have both iptables and firewalld on same host
+# firewalld is just a frontend for iptables - so we can't remove it
+ # if we try to stop non existing service ansible fails.
+# - name: Stop iptables, ip6tables services
+# service: name="{{ item }}" state=stopped
+# with_items:
+# - iptables
+# - ip6tables
+
+- name: Start firewalld service
+ service: name=firewalld state=started
+ become: yes
+
+- name: open firewall port 22 for SSH connections
+ firewalld: port="22/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: Install Airavata pre-requireties (RedHat)
+ yum: name={{ item }} state=latest update_cache=yes
+ with_items:
+ - git
+ - maven
+ become: yes
+ when: ansible_os_family == "RedHat"
+
+- name: Install Airavata pre-requireties (Debian)
+ apt: name={{ item }} state=latest update_cache=yes
+ with_items:
+ - git
+ - maven
+ become: yes
+ when: ansible_os_family == "Debian"
+
+- name: Create deployment directory {{ deployment_dir }}
+ become: yes
+ file: path={{ deployment_dir }}
+ state=directory
+ mode=0755
+ owner={{ user }}
+ group={{ group }}
+
+- name: Create source directory
+ become: yes
+ file: path={{airavata_source_dir}}
+ state=directory
+ mode=0755
+ owner={{ user }}
+ group={{ group }}
+
+- name: git checkout from airavata github repo {{ airavata_repo }} branch {{ git_branch }}
+ git: repo="{{ airavata_repo }}"
+ dest="{{ airavata_source_dir }}"
+ version="{{ git_branch }}"
+ register: checkout
+ tags: update
+ become: yes
+ become_user: "{{ user }}"
+
+- name: Run maven build
+ command: mvn clean install -Dmaven.test.skip=true chdir="{{ airavata_source_dir }}/"
+ environment:
+ MAVEN_OPTS: "-Xmx2048m"
+ register: build
+ tags: update
+ become: yes
+ become_user: "{{ user }}"
+...