You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2018/11/09 15:09:53 UTC
[airavata] branch develop updated: Parser Ansible
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/airavata.git
The following commit(s) were added to refs/heads/develop by this push:
new 05559e1 Parser Ansible
05559e1 is described below
commit 05559e1c2de805a4d4621f2036169a7e933e3ecc
Author: Dimuthu Wannipurage <di...@datasprouts.com>
AuthorDate: Fri Nov 9 10:09:44 2018 -0500
Parser Ansible
---
dev-tools/ansible/apiserver.yml | 4 +-
.../scigap/develop/group_vars/all/vars.yml | 9 +++-
.../ansible/roles/helix_setup/defaults/main.yml | 4 ++
dev-tools/ansible/roles/helix_setup/tasks/main.yml | 6 +++
.../parser-wm/airavata-server.properties.j2 | 60 ++++++++++++++++++++++
.../helix_setup/templates/parser-wm/logback.xml.j2 | 53 +++++++++++++++++++
6 files changed, 133 insertions(+), 3 deletions(-)
diff --git a/dev-tools/ansible/apiserver.yml b/dev-tools/ansible/apiserver.yml
index d1460b4..c893ac5 100644
--- a/dev-tools/ansible/apiserver.yml
+++ b/dev-tools/ansible/apiserver.yml
@@ -20,8 +20,8 @@
---
# Just gather facts on database
-- hosts: database
- gather_facts: True
+#- hosts: database
+# gather_facts: True
- hosts: api-orch
tags: api-orch, airavata
roles:
diff --git a/dev-tools/ansible/inventories/scigap/develop/group_vars/all/vars.yml b/dev-tools/ansible/inventories/scigap/develop/group_vars/all/vars.yml
index 2e16e12..64b7e8d 100644
--- a/dev-tools/ansible/inventories/scigap/develop/group_vars/all/vars.yml
+++ b/dev-tools/ansible/inventories/scigap/develop/group_vars/all/vars.yml
@@ -161,4 +161,11 @@ realtime_monitor_broker_topic: "helix-airavata-mq"
# Variables related to the broker in which parsed job statuses are published
job_monitor_broker_url: "{{ groups['kafka'][0] }}:{{ kafka_listener_port }}"
job_monitor_broker_topic: "parsed-data"
-job_monitor_broker_consumer_group: "MonitoringConsumer"
\ No newline at end of file
+job_monitor_broker_consumer_group: "MonitoringConsumer"
+
+# Variables related to parser framework
+parser_broker_url: "{{ groups['kafka'][0] }}:{{ kafka_listener_port }}"
+parser_broker_topic: "completed-messages"
+parser_broker_consumer_group: "ParsingConsumer"
+parser_storage_resource_id: "pgadev.scigap.org_7ddf28fd-d503-4ff8-bbc5-3279a7c3b99e"
+parser_broker_publisher_id: "ParserProducer"
\ No newline at end of file
diff --git a/dev-tools/ansible/roles/helix_setup/defaults/main.yml b/dev-tools/ansible/roles/helix_setup/defaults/main.yml
index 19222ce..dd91a09 100644
--- a/dev-tools/ansible/roles/helix_setup/defaults/main.yml
+++ b/dev-tools/ansible/roles/helix_setup/defaults/main.yml
@@ -23,21 +23,25 @@ helix_controller_name: "helixcontroller"
helix_participant_name: "helixparticipant"
helix_pre_wm_name: "prewm"
helix_post_wm_name: "postwm"
+helix_parser_wm_name: "parserwm"
helix_controller_version: "helix-controller-{{ snapshot_version }}"
helix_participant_version: "helix-participant-{{ snapshot_version }}"
helix_pre_wm_version: "helix-pre-wm-{{ snapshot_version }}"
helix_post_wm_version: "helix-post-wm-{{ snapshot_version }}"
+helix_parser_wm_version: "helix-parser-wm-{{ snapshot_version }}"
helix_controller_dist_name: "{{ helix_controller_version }}-bin.tar.gz"
helix_participant_dist_name: "{{ helix_participant_version }}-bin.tar.gz"
helix_pre_wm_dist_name: "{{ helix_pre_wm_version }}-bin.tar.gz"
helix_post_wm_dist_name: "{{ helix_post_wm_version }}-bin.tar.gz"
+helix_parser_wm_dist_name: "{{ helix_parser_wm_version }}-bin.tar.gz"
helix_controller_log_dir: "{{ helix_dir }}/{{ helix_controller_version }}/logs"
helix_participant_log_dir: "{{ helix_dir }}/{{ helix_participant_version }}/logs"
helix_pre_wm_dist_log_dir: "{{ helix_dir }}/{{ helix_pre_wm_version }}/logs"
helix_post_wm_dist_log_dir: "{{ helix_dir }}/{{ helix_post_wm_version }}/logs"
+helix_parser_wm_log_dir: "{{ helix_dir }}/{{ helix_parser_wm_version }}/logs"
job_status_publish_endpoint: "http://{{ groups['kafka'][0] }}:{{ kafka_rest_proxy_listener_port }}/topics/{{ realtime_monitor_broker_topic }}"
diff --git a/dev-tools/ansible/roles/helix_setup/tasks/main.yml b/dev-tools/ansible/roles/helix_setup/tasks/main.yml
index e8bdbc7..f632eed 100644
--- a/dev-tools/ansible/roles/helix_setup/tasks/main.yml
+++ b/dev-tools/ansible/roles/helix_setup/tasks/main.yml
@@ -50,6 +50,7 @@
- "{{ helix_participant_dist_name }}"
- "{{ helix_pre_wm_dist_name }}"
- "{{ helix_post_wm_dist_name }}"
+ - "{{ helix_parser_wm_dist_name }}"
# Copy properties files & logback.xml files
@@ -64,6 +65,7 @@
- { name: participant, dir: "{{ helix_participant_version }}" }
- { name: pre-wm, dir: "{{ helix_pre_wm_version }}" }
- { name: post-wm, dir: "{{ helix_post_wm_version }}" }
+ - { name: parser-wm, dir: "{{ helix_parser_wm_version }}" }
- name: Copy logback configuration file
template: "src={{ item.name }}/logback.xml.j2
@@ -76,6 +78,7 @@
- { name: participant, dir: "{{ helix_participant_version }}" }
- { name: pre-wm, dir: "{{ helix_pre_wm_version }}" }
- { name: post-wm, dir: "{{ helix_post_wm_version }}" }
+ - { name: parser-wm, dir: "{{ helix_parser_wm_version }}" }
- name: Create logs directory
file: "path={{ item }} state=directory owner={{ user }} group={{ group }}"
@@ -84,6 +87,7 @@
- "{{ helix_participant_log_dir }}"
- "{{ helix_pre_wm_dist_log_dir }}"
- "{{ helix_post_wm_dist_log_dir }}"
+ - "{{ helix_parser_wm_dist_log_dir }}"
- name: Stop daemons
command: "{{ item.command }} chdir={{ helix_dir }}/{{ item.dir }}/"
@@ -92,6 +96,7 @@
- { command: ./bin/participant-daemon.sh stop, dir: "{{ helix_participant_version }}" }
- { command: ./bin/pre-wm-daemon.sh stop, dir: "{{ helix_pre_wm_version }}" }
- { command: ./bin/post-wm-daemon.sh stop, dir: "{{ helix_post_wm_version }}" }
+ - { command: ./bin/parser-wm-daemon.sh stop, dir: "{{ helix_parser_wm_version }}" }
# Run bash-scripts
- name: Start daemons
@@ -101,6 +106,7 @@
- { command: ./bin/participant-daemon.sh start, dir: "{{ helix_participant_version }}" }
- { command: ./bin/pre-wm-daemon.sh start, dir: "{{ helix_pre_wm_version }}" }
- { command: ./bin/post-wm-daemon.sh start, dir: "{{ helix_post_wm_version }}" }
+ - { command: ./bin/parser-wm-daemon.sh start, dir: "{{ helix_parser_wm_version }}" }
#- name: systemd install helix setup service scripts
# template: src={{ item.template }}
diff --git a/dev-tools/ansible/roles/helix_setup/templates/parser-wm/airavata-server.properties.j2 b/dev-tools/ansible/roles/helix_setup/templates/parser-wm/airavata-server.properties.j2
new file mode 100644
index 0000000..9b7de96
--- /dev/null
+++ b/dev-tools/ansible/roles/helix_setup/templates/parser-wm/airavata-server.properties.j2
@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+###########################################################################
+# Registry Server Configurations
+###########################################################################
+regserver.server.host={{ registry_host }}
+regserver.server.port={{ registry_port }}
+
+###########################################################################
+# Helix workflow manager configurations
+###########################################################################
+helix.cluster.name={{ helix_cluster_name }}
+parser.workflow.manager.name={{ helix_parser_wm_name }}
+
+###########################################################################
+# AMQP Notification Configuration
+###########################################################################
+#for simple scenarios we can use the guest user
+#rabbitmq.broker.url=amqp://localhost:5672
+#for production scenarios, give url as amqp://userName:password@hostName:portNumber/virtualHost, create user, virtualhost
+# and give permissions, refer: http://blog.dtzq.com/2012/06/rabbitmq-users-and-virtual-hosts.html
+rabbitmq.broker.url={{ rabbitmq_broker_url }}
+rabbitmq.status.exchange.name=status_exchange
+rabbitmq.process.exchange.name=process_exchange
+rabbitmq.experiment.exchange.name=experiment_exchange
+durable.queue=false
+prefetch.count=200
+process.launch.queue.name=process.launch.queue
+experiment.launch..queue.name=experiment.launch.queue
+
+###########################################################################
+# Zookeeper Server Configuration
+###########################################################################
+zookeeper.server.connection={{ zookeeper_connection_url }}
+zookeeper.timeout=30000
+
+###########################################################################
+# Data Parser Configurations
+###########################################################################
+kafka.parsing.broker.url={{ parser_broker_url }}
+kafka.parser.broker.consumer.group={{ parser_broker_consumer_group }}
+kafka.parser.topic={{ parser_broker_topic }}
+parser.storage.resource.id={{ parser_storage_resource_id }}
+kafka.parsing.broker.publisher.id={{ parser_broker_publisher_id }}
diff --git a/dev-tools/ansible/roles/helix_setup/templates/parser-wm/logback.xml.j2 b/dev-tools/ansible/roles/helix_setup/templates/parser-wm/logback.xml.j2
new file mode 100644
index 0000000..4d5b241
--- /dev/null
+++ b/dev-tools/ansible/roles/helix_setup/templates/parser-wm/logback.xml.j2
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<configuration>
+
+ <appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %X - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <appender name="LOGFILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <File>{{ helix_parser_wm_log_dir }}/airavata.log</File>
+ <Append>true</Append>
+ <encoder>
+ <pattern>%d [%t] %-5p %c{30} %X - %m%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+ <fileNamePattern>{{ helix_controller_log_dir }}/airavata.log.%d{yyyy-MM-dd}</fileNamePattern>
+ <maxHistory>{{ helix_log_max_history }}</maxHistory>
+ <totalSizeCap>{{ helix_log_total_size_cap }}</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <logger name="ch.qos.logback" level="WARN"/>
+ <logger name="org.apache.helix" level="WARN"/>
+ <logger name="org.apache.zookeeper" level="ERROR"/>
+ <logger name="org.apache.airavata" level="INFO"/>
+ <logger name="org.hibernate" level="ERROR"/>
+ <root level="INFO">
+ <appender-ref ref="CONSOLE"/>
+ <appender-ref ref="LOGFILE"/>
+ </root>
+</configuration>