You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2020/03/10 16:30:32 UTC
[airavata] branch mft-integration updated: Ansible updates for
consul and mft installation
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch mft-integration
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/mft-integration by this push:
new cf1fca6 Ansible updates for consul and mft installation
cf1fca6 is described below
commit cf1fca6bdd1c19b54fbf979cb913a1c54e6636c7
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Tue Mar 10 12:30:18 2020 -0400
Ansible updates for consul and mft installation
---
dev-tools/ansible/consul.yml | 26 ++++++++
.../scigap/testing/group_vars/all/vars.yml | 6 +-
dev-tools/ansible/inventories/scigap/testing/hosts | 3 +
dev-tools/ansible/roles/consul/defaults/main.yml | 27 ++++++++
dev-tools/ansible/roles/consul/handlers/main.yml | 28 ++++++++
dev-tools/ansible/roles/consul/tasks/main.yml | 75 ++++++++++++++++++++++
.../roles/consul/templates/consul.config.j2 | 14 ++++
.../roles/consul/templates/consul.service.j2 | 10 +++
.../participant/airavata-server.properties.j2 | 9 ++-
dev-tools/ansible/roles/mft/defaults/main.yml | 27 +++++++-
dev-tools/ansible/roles/mft/tasks/main.yml | 8 +++
.../api-service/application.properties.j2 | 9 ++-
.../templates/controller/application.properties.j2 | 5 +-
13 files changed, 240 insertions(+), 7 deletions(-)
diff --git a/dev-tools/ansible/consul.yml b/dev-tools/ansible/consul.yml
new file mode 100644
index 0000000..9658338
--- /dev/null
+++ b/dev-tools/ansible/consul.yml
@@ -0,0 +1,26 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+- hosts: consul
+ roles:
+ - env_setup
+ - java
+ - consul
diff --git a/dev-tools/ansible/inventories/scigap/testing/group_vars/all/vars.yml b/dev-tools/ansible/inventories/scigap/testing/group_vars/all/vars.yml
index 9473ff1..9a2ce47 100644
--- a/dev-tools/ansible/inventories/scigap/testing/group_vars/all/vars.yml
+++ b/dev-tools/ansible/inventories/scigap/testing/group_vars/all/vars.yml
@@ -177,4 +177,8 @@ parser_broker_publisher_id: "ParserProducer"
thrift_client_pool_abandoned_removal_enabled: true
thrift_client_pool_abandoned_removal_logged: true
-usage_reporting_key: "NONE"
\ No newline at end of file
+usage_reporting_key: "NONE"
+
+mft_api_service_host: "{{ groups['mft'][0] }}"
+mft_api_service_grpc_port: 7004
+mft_api_service_http_port: 8088
\ No newline at end of file
diff --git a/dev-tools/ansible/inventories/scigap/testing/hosts b/dev-tools/ansible/inventories/scigap/testing/hosts
index bc5c3e0..b900d69 100644
--- a/dev-tools/ansible/inventories/scigap/testing/hosts
+++ b/dev-tools/ansible/inventories/scigap/testing/hosts
@@ -31,4 +31,7 @@
seagrid ansible_host=149.165.170.103
[mft]
+149.165.156.124 ansible_user=root
+
+[consul]
149.165.156.124 ansible_user=root
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/consul/defaults/main.yml b/dev-tools/ansible/roles/consul/defaults/main.yml
new file mode 100644
index 0000000..de79c61
--- /dev/null
+++ b/dev-tools/ansible/roles/consul/defaults/main.yml
@@ -0,0 +1,27 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+consul_version: 1.6.2
+consul_zip_file: consul_{{ consul_version }}_linux_amd64.zip
+consul_install_dir: /opt/consul/app
+consul_config_dir: /opt/consul/config
+consul_data_dir: /opt/consul/data
+consul_http_port: 8500
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/consul/handlers/main.yml b/dev-tools/ansible/roles/consul/handlers/main.yml
new file mode 100644
index 0000000..d6cd91e
--- /dev/null
+++ b/dev-tools/ansible/roles/consul/handlers/main.yml
@@ -0,0 +1,28 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+# handlers file for consul-server
+- name: restart consul
+ systemd:
+ name: consul.service
+ daemon_reload: yes
+ state: restarted
+ become: yes
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/consul/tasks/main.yml b/dev-tools/ansible/roles/consul/tasks/main.yml
new file mode 100644
index 0000000..b3827a6
--- /dev/null
+++ b/dev-tools/ansible/roles/consul/tasks/main.yml
@@ -0,0 +1,75 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+# Create the directory for Consul deployment
+- name: Create consul deployment directory
+ file: path="{{ consul_install_dir }}" state=directory owner={{ user }} group={{ group }} recurse=yes
+
+- name: Create consul config directory
+ file: path="{{ consul_config_dir }}" state=directory owner={{ user }} group={{ group }} recurse=yes
+
+- name: Create consul data directory
+ file: path="{{ consul_data_dir }}" state=directory owner={{ user }} group={{ group }} recurse=yes
+
+# tasks file for consul-server
+- name: get consul zip
+ get_url:
+ dest: "/tmp/{{ consul_zip_file }}"
+ url: "https://releases.hashicorp.com/consul/{{ consul_version }}/{{ consul_zip_file }}"
+ become: yes
+
+- name: place unzipped consul on path
+ unarchive:
+ src: "/tmp/{{ consul_zip_file }}"
+ dest: "{{ consul_install_dir }}"
+ remote_src: yes
+ become: yes
+
+- name: ensure directories for data and config exists
+ file:
+ path: "{{ item }}"
+ state: directory
+ with_items:
+ - "{{ consul_config_dir }}"
+ - "{{ consul_data_dir }}"
+ become: yes
+
+- name: send consul configuration file
+ template:
+ dest: "{{ consul_config_dir }}/config.json"
+ src: consul.config.j2
+ notify: restart consul
+ become: yes
+
+- name: Open firwall ports
+ firewalld: port={{ item }} zone=public permanent=true state=enabled immediate=yes
+ with_items:
+ - "{{ consul_http_port }}/tcp"
+ become_user: root
+
+- name: ensure consul service file exists
+ template:
+ dest: /etc/systemd/system/consul.service
+ src: consul.service.j2
+ force: yes
+ mode: 0644
+ notify: restart consul
+ become: yes
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/consul/templates/consul.config.j2 b/dev-tools/ansible/roles/consul/templates/consul.config.j2
new file mode 100644
index 0000000..9b20964
--- /dev/null
+++ b/dev-tools/ansible/roles/consul/templates/consul.config.j2
@@ -0,0 +1,14 @@
+{
+ "node_name": "consul-mft",
+ "addresses": {
+ "http": "{{ hostvars[groups['consul'][0]]['ansible_default_ipv4']['address'] }} 127.0.0.1"
+ },
+ "server": true,
+ "advertise_addr": "{{ hostvars[groups['consul'][0]]['ansible_default_ipv4']['address'] }}",
+ "client_addr": "127.0.0.1 {{ hostvars[groups['consul'][0]]['ansible_default_ipv4']['address'] }}",
+ "connect": {
+ "enabled": true
+ },
+ "data_dir": "{{ consul_data_dir }}",
+ "bootstrap": true
+}
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/consul/templates/consul.service.j2 b/dev-tools/ansible/roles/consul/templates/consul.service.j2
new file mode 100644
index 0000000..b0a4e39
--- /dev/null
+++ b/dev-tools/ansible/roles/consul/templates/consul.service.j2
@@ -0,0 +1,10 @@
+[Unit]
+Description=solo consul server example
+
+[Service]
+WorkingDirectory={{ consul_config_dir }}
+User=root
+ExecStart={{ consul_install_dir }}/consul agent -config-dir={{ consul_config_dir }}
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2 b/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
index 5f842b5..2431972 100644
--- a/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
+++ b/dev-tools/ansible/roles/helix_setup/templates/participant/airavata-server.properties.j2
@@ -106,4 +106,11 @@ thrift.client.pool.abandoned.removal.logged={{ thrift_client_pool_abandoned_remo
###########################################################################
usage.reporting.key={{ usage_reporting_key }}
-usage.reporting.endpoint=https://xsede-xdcdb-api.xsede.org/gateway/v2/job_attributes
\ No newline at end of file
+usage.reporting.endpoint=https://xsede-xdcdb-api.xsede.org/gateway/v2/job_attributes
+
+###########################################################################
+# Managed File Transfer Configurations
+###########################################################################
+
+mft.server.host={{ mft_api_service_host }}
+mft.server.port={{ mft_api_service_grpc_port }}
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/mft/defaults/main.yml b/dev-tools/ansible/roles/mft/defaults/main.yml
index 0b101e6..30fd725 100644
--- a/dev-tools/ansible/roles/mft/defaults/main.yml
+++ b/dev-tools/ansible/roles/mft/defaults/main.yml
@@ -1,3 +1,23 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
apache_maven_version: "apache-maven-3.6.3"
apache_maven_url: "https://www-eu.apache.org/dist/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz"
@@ -16,12 +36,13 @@ mft_resource_server_version: "Resource-Service-0.01"
mft_secret_server_version: "Secret-Service-0.01"
mft_api_server_version: "API-Service-0.01"
+secret_service_host: "{{ hostvars[groups['mft'][0]]['ansible_default_ipv4']['address'] }}"
secret_service_grpc_port: 7003
secret_service_http_port: 8081
+resource_service_host: "{{ hostvars[groups['mft'][0]]['ansible_default_ipv4']['address'] }}"
resource_service_grpc_port: 7002
resource_service_http_port: 8080
-mft_api_service_http_port: 8088
-mft_api_service_grpc_port: 7004
-
+consul_host: "{{ hostvars[groups['consul'][0]]['ansible_default_ipv4']['address'] }}"
+consul_port: 8500
diff --git a/dev-tools/ansible/roles/mft/tasks/main.yml b/dev-tools/ansible/roles/mft/tasks/main.yml
index 78ea7ae..62b80e3 100644
--- a/dev-tools/ansible/roles/mft/tasks/main.yml
+++ b/dev-tools/ansible/roles/mft/tasks/main.yml
@@ -144,6 +144,14 @@
become: yes
become_user: airavata
+- name: Open firwall ports
+ firewalld: port={{ item }} zone=public permanent=true state=enabled immediate=yes
+ with_items:
+ - "{{ secret_service_grpc_port }}/tcp"
+ - "{{ resource_service_grpc_port }}/tcp"
+ - "{{ mft_api_service_grpc_port }}/tcp"
+ become_user: root
+
- name: Start daemons running
command: "{{ item.command }} chdir={{ mft_distribution_dir }}/{{ item.dir }}/"
with_items:
diff --git a/dev-tools/ansible/roles/mft/templates/api-service/application.properties.j2 b/dev-tools/ansible/roles/mft/templates/api-service/application.properties.j2
index 4704cc4..bd674e4 100644
--- a/dev-tools/ansible/roles/mft/templates/api-service/application.properties.j2
+++ b/dev-tools/ansible/roles/mft/templates/api-service/application.properties.j2
@@ -16,4 +16,11 @@
#
server.port={{ mft_api_service_http_port }}
-grpc.port={{ mft_api_service_grpc_port }}
\ No newline at end of file
+grpc.port={{ mft_api_service_grpc_port }}
+consul.host={{ consul_host }}
+consul.port={{ consul_port }}
+
+resource.service.host={{ resource_service_host }}
+resource.service.port={{ resource_service_grpc_port }}
+secret.service.host={{ secret_service_host }}
+secret.service.port={{ secret_service_grpc_port }}
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/mft/templates/controller/application.properties.j2 b/dev-tools/ansible/roles/mft/templates/controller/application.properties.j2
index a9fd83f..1a18de0 100644
--- a/dev-tools/ansible/roles/mft/templates/controller/application.properties.j2
+++ b/dev-tools/ansible/roles/mft/templates/controller/application.properties.j2
@@ -13,4 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
\ No newline at end of file
+#
+
+consul.host={{ consul_host }}
+consul.port={{ consul_port }}
\ No newline at end of file