You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by sc...@apache.org on 2016/11/04 15:18:44 UTC
[2/3] airavata git commit: Airavata ansible scripts
Airavata ansible scripts
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/56384a1a
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/56384a1a
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/56384a1a
Branch: refs/heads/develop
Commit: 56384a1a028acf50e3111234d0ab5f15c8aadf70
Parents: 02021d3
Author: Shameera Rathnayaka <sh...@gmail.com>
Authored: Thu Jul 28 12:12:54 2016 -0400
Committer: scnakandala <su...@gmail.com>
Committed: Fri Nov 4 11:04:38 2016 -0400
----------------------------------------------------------------------
group_vars/all | 51 ++++
hosts | 18 ++
.../files/mysql-connector-java-5.1.37-bin.jar | Bin 0 -> 985603 bytes
roles/api-orch/handlers/main.yml | 10 +
roles/api-orch/tasks/main.yml | 54 ++++
.../templates/airavata-server.properties.j2 | 278 +++++++++++++++++++
roles/api-orch/templates/gfac-config.yaml.j2 | 111 ++++++++
roles/api-orch/vars/main.yml | 3 +
roles/common/files/airavata.jks | Bin 0 -> 1410 bytes
roles/common/files/client_truststore.jks | Bin 0 -> 3558 bytes
roles/common/handlers/main.yml | 45 +++
roles/common/tasks/main.yml | 40 +++
roles/common/vars/main.yml | 5 +
.../files/mysql-connector-java-5.1.37-bin.jar | Bin 0 -> 985603 bytes
roles/gfac/handlers/main.yml | 8 +
roles/gfac/tasks/main.yml | 53 ++++
.../templates/airavata-server.properties.j2 | 278 +++++++++++++++++++
roles/gfac/templates/gfac-config.yaml.j2 | 111 ++++++++
roles/gfac/vars/main.yml | 3 +
roles/rabbitmq/handlers/main.yml | 10 +
roles/rabbitmq/tasks/main.yml | 44 +++
roles/setup/tasks/main.yml | 44 +++
roles/setup/tasks/redhat.yml | 12 +
roles/setup/vars/main.yml | 15 +
roles/zookeeper/handlers/main.yml | 19 ++
roles/zookeeper/tasks/main.yml | 15 +
roles/zookeeper/templates/zoo.cfg.j2 | 28 ++
roles/zookeeper/vars/main.yml | 17 ++
site.yml | 25 ++
29 files changed, 1297 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/group_vars/all
----------------------------------------------------------------------
diff --git a/group_vars/all b/group_vars/all
new file mode 100644
index 0000000..2c2cb95
--- /dev/null
+++ b/group_vars/all
@@ -0,0 +1,51 @@
+---
+ansible_connection: ssh
+ansible_ssh_user: root
+ansible_ssh_private_key_file: /Users/syodage/Projects/scigap/JetCloud/jetcloud.key
+
+user: airavata
+group: airavata
+user_home: "/home/{{ user }}"
+deployment_dir: "{{ user_home }}/master-deployment"
+
+airavata_dist: "apache-airavata-server-0.17-SNAPSHOT"
+airavata_dist_name: "{{ airavata_dist }}-bin.tar.gz"
+
+db_server: "gw62.iu.xsede.org"
+db_user: "SciGaP15Deploy"
+db_password: "xxxxxxxxxxx"
+app_catalog: "app_catalog"
+exp_catalog: "exp_catalog"
+replica_catalog: "replica_catalog"
+workflow_catalog: "wf_catalog"
+credential_store: "credential_store"
+
+rabbitmq_server: "localhost"
+rabbitmq_vhost: "master"
+rabbitmq_user: "airavata"
+rabbitmq_password: "airavata"
+rabbitmq_broker_url: "amqp://{{ rabbitmq_user }}:{{ rabbitmq_password }}@{{ rabbitmq_server}}:5672/{{ rabbitmq_vhost }}"
+
+key_store: "airavata.jks"
+cred_key_store: "client_truststore.jks"
+cred_store_server_host: "gw77.iu.xsede.org"
+key_stores_location: "{{ deployment_dir }}/keystores"
+
+mysql_connector_jar: "mysql-connector-java-5.1.37-bin.jar"
+
+api_server_name: "apiserver-node0"
+api_server_host: "{{ host_address }}"
+api_server_port: "8930"
+
+orchestrator_name: "orchestrator-node0"
+orchestrator_host: "{{ host_address }}"
+orchestrator_port: "8940"
+
+gfac_name: "gfac-node0"
+gfac_host: "{{ host_address }}"
+gfac_port: "8950"
+
+monitor_email_address: "jobs@scigap.org"
+monitor_email_password: "xxxxx"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/hosts
----------------------------------------------------------------------
diff --git a/hosts b/hosts
new file mode 100644
index 0000000..11a03c7
--- /dev/null
+++ b/hosts
@@ -0,0 +1,18 @@
+# inventory file : production
+
+[api-orch]
+149.165.156.196 host_name=js-156-196 host_address=149.165.156.196
+#149.165.171.11 host_name=js-171-11 host_address=149.165.171.11
+
+[gfac]
+149.165.156.196 host_name=js-156-196 host_address=149.165.156.196
+#149.165.171.11 host_name=js-171-11 host_address=149.165.171.11
+
+[pga]
+149.165.156.196
+
+[zookeeper]
+149.165.156.196 host_name=js-156-196 host_address=149.165.156.196
+
+[rabbitmq]
+149.165.156.196 host_name=js-156-196 host_address=149.165.156.196
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/api-orch/files/mysql-connector-java-5.1.37-bin.jar
----------------------------------------------------------------------
diff --git a/roles/api-orch/files/mysql-connector-java-5.1.37-bin.jar b/roles/api-orch/files/mysql-connector-java-5.1.37-bin.jar
new file mode 100644
index 0000000..465af67
Binary files /dev/null and b/roles/api-orch/files/mysql-connector-java-5.1.37-bin.jar differ
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/api-orch/handlers/main.yml
----------------------------------------------------------------------
diff --git a/roles/api-orch/handlers/main.yml b/roles/api-orch/handlers/main.yml
new file mode 100644
index 0000000..c1a828d
--- /dev/null
+++ b/roles/api-orch/handlers/main.yml
@@ -0,0 +1,10 @@
+---
+
+# Api server, Orchestrator related handlers
+- name: start api-orch
+ command: ./bin/airavata-server-start.sh api-orch -d chdir="{{ api_orch_dir }}/{{ airavata_dist }}/" creates="{{ api_orch_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+- name: stop api-orch
+ command: ./bin/airavata-server-stop.sh -f chdir="{{ api_orch_dir }}/{{ airavata_dist }}/" removes="{{ api_orch_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/api-orch/tasks/main.yml
----------------------------------------------------------------------
diff --git a/roles/api-orch/tasks/main.yml b/roles/api-orch/tasks/main.yml
new file mode 100644
index 0000000..827f520
--- /dev/null
+++ b/roles/api-orch/tasks/main.yml
@@ -0,0 +1,54 @@
+---
+################################################################################
+# api-orch deployment
+- name: Create api-orchestrator deployment directory
+ file: path="{{ api_orch_dir }}" state=directory owner="{{ user }}" group="{{ group }}"
+ when: build|success
+ tags: deploy, update_api_orch, update
+
+- name: Check previous deployments
+ stat: path="{{ api_orch_dir }}/{{ airavata_dist }}" get_md5=no get_checksum=no
+ register: pp
+ tags: deploy, update_api_orch, update
+
+- block:
+ - name: stop api-orch
+ command: ./bin/airavata-server-stop.sh -f chdir="{{ api_orch_dir }}/{{ airavata_dist }}/" removes="{{ api_orch_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+ - name: Delete previous deployments
+ file: path="{{ gfac_dir }}/{{ airavata_dist }}" state=absent
+
+ when: pp.stat.exists == True
+ tags: deploy, update_api_orch, update
+#
+# - block:
+# - name: Remove pervious backup directory
+# file: path="{{ api_orch_dir }}/backup" state=absent
+# notify:
+# - stop api-orch
+#
+# - name: Backup previous deployment directory
+# copy: src="{{ api_orch_dir }}/{{ airavata_dist }}" dest="{{ api_orch_dir }}/backup" copy=no
+#
+# when: p.stat.exist == True
+# tags: deploy update_api_orch update
+
+- name: Copy distribution to api-orcheatrator deployment directory
+ unarchive: src="{{ airavata_source_dir }}/distribution/target/{{ airavata_dist_name }}" dest="{{ api_orch_dir }}/" copy=no
+ become: yes
+ become_user: "{{ user }}"
+ tags: deploy, update_api_orch, update
+
+- name: Copy Airavata server properties file
+ template: src=airavata-server.properties.j2 dest="{{ api_orch_dir }}/{{ airavata_dist }}/bin/airavata-server.properties" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ tags: deploy, update_api_orch, update
+
+- name: Copy Gfac configuration file
+ template: src=gfac-config.yaml.j2 dest="{{ api_orch_dir }}/{{ airavata_dist }}/bin/gfac-config.yaml" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ tags: deploy, update_api_orch, update
+
+- name: Copy Mysql jar to lib
+ copy: src="{{ mysql_connector_jar }}" dest="{{ api_orch_dir }}/{{ airavata_dist }}/lib/{{ mysql_connector_jar }}" owner="{{ user }}" group="{{ group }}"
+ tags: deploy, update_api_orch, update
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/api-orch/templates/airavata-server.properties.j2
----------------------------------------------------------------------
diff --git a/roles/api-orch/templates/airavata-server.properties.j2 b/roles/api-orch/templates/airavata-server.properties.j2
new file mode 100644
index 0000000..449a7fd
--- /dev/null
+++ b/roles/api-orch/templates/airavata-server.properties.j2
@@ -0,0 +1,278 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+###########################################################################
+#
+# This properties file provides configuration for all Airavata Services:
+# API Server, Registry, Workflow Interpreter, GFac, Orchestrator
+#
+###########################################################################
+
+###########################################################################
+# API Server Registry Configuration
+###########################################################################
+
+#for derby [AiravataJPARegistry]
+#registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#registry.jdbc.url=jdbc:derby://localhost:1527/experiment_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+registry.jdbc.driver=com.mysql.jdbc.Driver
+registry.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ exp_catalog }}
+registry.jdbc.user={{ db_user }}
+registry.jdbc.password={{ db_password }}
+#FIXME: Probably the following property should be removed.
+start.derby.server.mode=false
+validationQuery=SELECT 1 from CONFIGURATION
+cache.enable=false
+jpa.cache.size=-1
+#jpa.connection.properties=MaxActive=10,MaxIdle=5,MinIdle=2,MaxWait=60000,testWhileIdle=true,testOnBorrow=true
+
+# Properties for default user mode
+default.registry.user=admin
+default.registry.password=admin
+default.registry.password.hash.method=SHA
+default.registry.gateway=default
+
+###########################################################################
+# Application Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+#appcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#appcatalog.jdbc.url=jdbc:derby://localhost:1527/app_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+appcatalog.jdbc.driver=com.mysql.jdbc.Driver
+appcatalog.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ app_catalog }}
+appcatalog.jdbc.user={{ db_user }}
+appcatalog.jdbc.password={{ db_password }}
+appcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+##########################################################################
+# Replica Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+#replicacatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#replicacatalog.jdbc.url=jdbc:derby://localhost:1527/replica_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+replicacatalog.jdbc.driver=com.mysql.jdbc.Driver
+replicacatalog.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ replica_catalog }}
+replicacatalog.jdbc.user={{ db_user }}
+replicacatalog.jdbc.password={{ db_password }}
+replicacatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+###########################################################################
+# Workflow Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+#workflowcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#workflowcatalog.jdbc.url=jdbc:derby://localhost:1527/workflow_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+workflowcatalog.jdbc.driver=com.mysql.jdbc.Driver
+workflowcatalog.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ replica_catalog }}
+workflowcatalog.jdbc.user={{ db_user }}
+workflowcatalog.jdbc.password={{ db_password }}
+workflowcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+###########################################################################
+# User Profile MongoDB Configuration
+###########################################################################
+userprofile.mongodb.host=localhost
+userprofile.mongodb.port=27017
+
+
+###########################################################################
+# Server module Configuration
+###########################################################################
+#credential store server should be started before API server
+#This is obsolete property with new script files.
+#servers=credentialstore,apiserver,orchestrator
+
+
+###########################################################################
+# API Server Configurations
+###########################################################################
+apiserver.name={{ api_server_name }}
+apiserver.host={{ api_server_host }}
+apiserver.port={{ api_server_port }}
+apiserver.min.threads=50
+
+###########################################################################
+# API Server SSL Configurations
+###########################################################################
+apiserver.tls.enabled=true
+apiserver.keystore={{ key_stores_location }}/{{ key_store }}
+apiserver.keystore.password=airavata
+
+###########################################################################
+# Orchestrator Server Configurations
+###########################################################################
+orchestrator.server.name={{ orchestrator_name }}
+orchestrator.server.host={{ orchestrator_host }}
+orchestrator.server.port={{ orchestrator_port }}
+orchestrator.server.min.threads=50
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+submitter.interval=10000
+threadpool.size=10
+start.submitter=true
+embedded.mode=true
+enable.validation=true
+
+
+###########################################################################
+# GFac Server Configurations
+###########################################################################
+gfac.server.name={{ gfac_name }}
+gfac.server.host={{ gfac_host }}
+gfac.server.port={{ gfac_port }}
+gfac.thread.pool.size=50
+host.scheduler=org.apache.airavata.gfac.impl.DefaultHostScheduler
+
+
+
+###########################################################################
+# Airavata Workflow Interpreter Configurations
+###########################################################################
+workflowserver=org.apache.airavata.api.server.WorkflowServer
+enactment.thread.pool.size=10
+
+#to define custom workflow parser user following property
+#workflow.parser=org.apache.airavata.workflow.core.parser.AiravataWorkflowBuilder
+
+
+
+###########################################################################
+# Job Scheduler can send informative email messages to you about the status of your job.
+# Specify a string which consists of either the single character "n" (no mail), or one or more
+# of the characters "a" (send mail when job is aborted), "b" (send mail when job begins),
+# and "e" (send mail when job terminates). The default is "a" if not specified.
+###########################################################################
+
+job.notification.enable=true
+#Provide comma separated email ids as a string if more than one
+job.notification.emailids=
+job.notification.flags=abe
+
+###########################################################################
+# Credential Store module Configuration
+###########################################################################
+credential.store.keystore.url=/home/airavata/airavata_sym.jks
+credential.store.keystore.alias=airavata
+credential.store.keystore.password=airavata
+credential.store.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ credential_store }}
+credential.store.jdbc.user={{ db_user }}
+credential.store.jdbc.password={{ db_password }}
+credential.store.jdbc.driver=com.mysql.jdbc.Driver
+credential.store.server.host={{ cred_store_server_host }}
+credential.store.server.port=8960
+credentialstore=org.apache.airavata.credential.store.server.CredentialStoreServer
+
+# these properties used by credential store email notifications
+email.server=smtp.googlemail.com
+email.server.port=465
+email.user=airavata
+email.password=xxx
+email.ssl=true
+email.from=airavata@apache.org
+
+# SSH PKI key pair or ssh password can be used SSH based sshKeyAuthentication is used.
+# if user specify both password sshKeyAuthentication gets the higher preference
+
+################# ---------- For ssh key pair sshKeyAuthentication ------------------- ################
+#ssh.public.key=/path to public key for ssh
+#ssh.private.key=/path to private key file for ssh
+#ssh.keypass=passphrase for the private key
+#ssh.username=username for ssh connection
+## If you set "yes" for ssh.strict.hostKey.checking, then you must provide known hosts file path
+#ssh.strict.hostKey.checking=yes/no
+#ssh.known.hosts.file=/path to known hosts file
+### Incase of password sshKeyAuthentication.
+#ssh.password=Password for ssh connection
+
+################ ---------- BES Properties ------------------- ###############
+#bes.ca.cert.path=<location>/certificates/cacert.pem
+#bes.ca.key.path=<location>/certificates/cakey.pem
+#bes.ca.key.pass=passphrase
+
+###########################################################################
+# Monitoring module Configuration
+###########################################################################
+
+#This will be the primary monitoring tool which runs in airavata, in future there will be multiple monitoring
+#mechanisms and one would be able to start a monitor
+monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor
+
+#These properties will used to enable email base monitoring
+email.based.monitor.host=imap.gmail.com
+email.based.monitor.address={{ monitor_email_address }}
+email.based.monitor.password={{ monitor_email_password }}
+email.based.monitor.folder.name=INBOX
+# either imaps or pop3
+email.based.monitor.store.protocol=imaps
+#These property will be used to query the email server periodically. value in milliseconds(ms).
+email.based.monitoring.period=10000
+
+###########################################################################
+# AMQP Notification Configuration
+###########################################################################
+#for simple scenarios we can use the guest user
+#rabbitmq.broker.url=amqp://localhost:5672
+#for production scenarios, give url as amqp://userName:password@hostName:portNumber/virtualHost, create user, virtualhost
+# and give permissions, refer: http://blog.dtzq.com/2012/06/rabbitmq-users-and-virtual-hosts.html
+rabbitmq.broker.url={{ rabbitmq_broker_url }}
+status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
+task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQProcessLaunchPublisher
+rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
+durable.queue=false
+prefetch.count=200
+launch.queue.name=launch.queue
+cancel.queue.name=cancel.queue
+activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
+rabbitmq.exchange.name=airavata_rabbitmq_exchange
+
+###########################################################################
+# Zookeeper Server Configuration
+###########################################################################
+embedded.zk=false
+zookeeper.server.connection=localhost:2181
+zookeeper.timeout=30000
+
+########################################################################
+## API Security Configuration
+########################################################################
+api.secured=true
+security.manager.class=org.apache.airavata.api.server.security.DefaultAiravataSecurityManager
+### TLS related configuration ####
+TLS.enabled=true
+TLS.api.server.port=9930
+TLS.client.timeout=10000
+#### keystore configuration ####
+keystore.path={{ key_stores_location }}/{{ key_store }}
+keystore.password=airavata
+#### trust store configuration ####
+trust.store={{ key_stores_location }}/{{ cred_key_store }}
+trust.store.password=airavata
+#### remote authorization server url ####
+remote.oauth.authorization.server=https://idp.scigap.org:9443/services/
+#### xacml based authorization policy ####
+authorization.policy=airavata-default-xacml-policy
+#### authorization cache related configuration ####
+authz.cache.enabled=true
+authz.cache.manager.class=org.apache.airavata.api.server.security.authzcache.DefaultAuthzCacheManager
+in.memory.cache.size=1000
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/api-orch/templates/gfac-config.yaml.j2
----------------------------------------------------------------------
diff --git a/roles/api-orch/templates/gfac-config.yaml.j2 b/roles/api-orch/templates/gfac-config.yaml.j2
new file mode 100644
index 0000000..3df5832
--- /dev/null
+++ b/roles/api-orch/templates/gfac-config.yaml.j2
@@ -0,0 +1,111 @@
+##################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+################################################################
+
+jobSubmitters:
+ - submissionProtocol: SSH
+ taskClass: org.apache.airavata.gfac.impl.task.DefaultJobSubmissionTask
+# properties:
+# - userName: airavata
+# passPhrase: airavata
+# privateKey: /path/to/the/privatekey
+# publicKey: /path/to/the/publickey
+# hostName: remote.client.hostName
+
+ - submissionProtocol: SSH_FORK
+ taskClass: org.apache.airavata.gfac.impl.task.ForkJobSubmissionTask
+
+ - submissionProtocol: LOCAL
+ taskClass: org.apache.airavata.gfac.impl.task.DefaultJobSubmissionTask
+
+# Following job subbmitters are not yet implemented.
+
+ # - submissionProtocol: GSISSH
+ # taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask
+commonTasks:
+ - type: SETUP
+ taskClass: org.apache.airavata.task.common.SetupTask
+
+ - type: CLEANUP
+ taskClass: org.apache.airavata.task.common.CleanupTask
+
+fileTransferTasks:
+# - transferProtocol: SCP
+# taskClass: org.apache.airavata.gfac.impl.task.DataStageTask
+
+# If your client doen't run the same instance where airavata server is running then you need to comment above
+# DataStageTask and uncomment SCPDataStageTask. To work with SCPDataStageTask, you either need to
+# provide ssh keys or password.
+
+ - transferProtocol: SCP
+ taskClass: org.apache.airavata.gfac.impl.task.SCPDataStageTask
+ properties:
+ - userName: root
+ passPhrase: ultrascan
+ privateKeyPath: /home/airavata/.ssh/id_rsa
+ publicKeyPath: /home/airavata/.ssh/id_rsa.pub
+ hostName: gw75.iu.xsede.org
+ inputPath: /var/www/portal/experimentData/
+# password: password
+
+# Following transfer tasks are not yet implemented.
+ #- transferProtocol: SFTP
+ # taskClass: org.apache.airavata.task.adapters.SFTPFileTransferTask
+
+ #- transferProtocol: GRIDFTP
+ # taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask
+
+ #- transferProtocol: LOCAL
+ # taskClass: org.apache.airavata.task.adapters.LocalFileTransferTask
+
+resources:
+ - jobManagerType: PBS
+ commandOutputParser: org.apache.airavata.gfac.impl.job.PBSOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser
+ resourceEmailAddresses:
+ - pbsconsult@sdsc.edu # gordon
+ - adm@trident.bigred2.uits.iu.edu # Bigred2
+ - root <ad...@trident.bigred2.uits.iu.edu> # Bigred2
+ - root <ad...@scyld.localdomain> # alamo
+ - root <ad...@m2.karst.uits.iu.edu> #karst
+
+ - jobManagerType: SLURM
+ commandOutputParser: org.apache.airavata.gfac.impl.job.SlurmOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser
+ resourceEmailAddresses:
+ - SDSC Admin <sl...@comet-fe3.sdsc.edu> # comet
+ - slurm@batch1.stampede.tacc.utexas.edu # stampede
+ - SDSC Admin <sl...@comet-fe4.sdsc.edu> # comet new
+ - Slurm <sl...@psc.edu> # bridges
+ - Slurm Daemon <sl...@odin.oscer.ou.edu> # OU Schooner
+
+ - jobManagerType: UGE
+ commandOutputParser: org.apache.airavata.gfac.impl.job.UGEOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.UGEEmailParser
+ resourceEmailAddresses:
+ - ls4.tacc.utexas.edu # contain Lonestar
+
+ - jobManagerType: LSF
+ commandOutputParser: org.apache.airavata.gfac.impl.job.LSFOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.LSFEmailParser
+ resourceEmailAddresses:
+ - iu.xsede.edu # test resource mail address
+
+ - jobManagerType: FORK
+ commandOutputParser: org.apache.airavata.gfac.impl.job.ForkOutputParser
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/api-orch/vars/main.yml
----------------------------------------------------------------------
diff --git a/roles/api-orch/vars/main.yml b/roles/api-orch/vars/main.yml
new file mode 100644
index 0000000..16b85b0
--- /dev/null
+++ b/roles/api-orch/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# API Server and Orchestrator related variables
+api_orch_dir: "{{ deployment_dir }}/api-orchestrator"
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/common/files/airavata.jks
----------------------------------------------------------------------
diff --git a/roles/common/files/airavata.jks b/roles/common/files/airavata.jks
new file mode 100644
index 0000000..e0e46e2
Binary files /dev/null and b/roles/common/files/airavata.jks differ
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/common/files/client_truststore.jks
----------------------------------------------------------------------
diff --git a/roles/common/files/client_truststore.jks b/roles/common/files/client_truststore.jks
new file mode 100644
index 0000000..4d39906
Binary files /dev/null and b/roles/common/files/client_truststore.jks differ
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/common/handlers/main.yml
----------------------------------------------------------------------
diff --git a/roles/common/handlers/main.yml b/roles/common/handlers/main.yml
new file mode 100644
index 0000000..d5c99e94
--- /dev/null
+++ b/roles/common/handlers/main.yml
@@ -0,0 +1,45 @@
+---
+# Handler file
+# Can have multiple handlers
+
+# Zookeeper related handlers
+- name: start zookeeper
+ command: ./bin/zkServer.sh start chdir={{zookeeper_dir}}/ creates={{zookeeper_data_dir}}/zookeeper_server.pid
+ become: true
+ become_user: "{{ user }}"
+
+- name: stop zookeeper
+ command: ./bin/zkServer.sh stop chdir={{zookeeper_dir}}/ removes={{zookeeper_data_dir}}/zookeeper_server.pid
+ become: true
+ become_user: "{{ user }}"
+
+- name: restart zookeeper
+ command: ./bin/zkServer.sh restart chdir={{zookeeper_dir}}/ removes={{zookeeper_data_dir}}/zookeeper_server.pid
+ become: true
+ become_user: "{{ user }}"
+
+# Rabbitmq related handlers
+- name: start rabbitmq
+ service: name=rabbitmq-server state=started
+
+- name: stop rabbitmq
+ service: name=rabbitmq-server state=stopped
+
+- name: restart rabbitmq
+ service: name=rabbitmq-server state=restarted
+
+# Gfac related handlers
+- name: start gfac
+ command: ./bin/airavata-server-start.sh gfac -d chdir="{{ gfac_dir }}/{{ airavata_dist }}/" creates="{{ gfac_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+- name: stop gfac
+ command: ./bin/airavata-server-stop.sh -f chdir="{{ gfac_dir }}/{{ airavata_dist }}/" removes="{{ gfac_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+# Gfac related handlers
+- name: start api-orch
+ command: ./bin/airavata-server-start.sh api-orch -d chdir="{{ api_orch_dir }}/{{ airavata_dist }}/" creates="{{ api_orch_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+- name: stop api-orch
+ command: ./bin/airavata-server-stop.sh -f chdir="{{ api_orch_dir }}/{{ airavata_dist }}/" removes="{{ api_orch_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/common/tasks/main.yml
----------------------------------------------------------------------
diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml
new file mode 100644
index 0000000..18d628d
--- /dev/null
+++ b/roles/common/tasks/main.yml
@@ -0,0 +1,40 @@
+---
+# Setup airavata source
+- name: Create deployment directory {{ deployment_dir }}
+ file: path={{ deployment_dir }} state=directory mode=0755
+ become: true
+ become_user: "{{ user }}"
+ tags: deploy
+
+- name: Create source directory
+ file: path={{airavata_source_dir}} state=directory mode=0755 owner={{ user }} group={{ group }}
+ tags: deploy
+
+- name: git checkout from airavata github
+ git: repo=https://git-wip-us.apache.org/repos/asf/airavata.git dest="{{ airavata_source_dir }}" version=master
+ register: checkout
+ become: true
+ become_user: "{{ user }}"
+ tags: deploy, update_gfac, update_api_orch, update
+
+- name: Run maven build
+ command: mvn clean install -Dmaven.test.skip=true chdir="{{ airavata_source_dir }}/"
+ when: (checkout|success) or (checkout|skipped)
+ register: build
+ become: true
+ become_user: "{{ user }}"
+ tags: deploy, update_gfac, update_api_orch, update
+
+################################################################################
+# copy key store and trust store files
+- name: Create KeyStores directory
+ file: path="{{ key_stores_location }}" state=directory owner="{{ user }}" group="{{ group }}"
+ tags: deploy
+
+- name: Transfer airavata.jks KeyStore file
+ copy: src="{{ key_store }}" dest="{{ key_stores_location }}/{{ key_store }}" owner="{{ user }}" group="{{ group }}"
+ tags: deploy
+
+- name: Transfer client trust store KeyStore file
+ copy: src="{{ cred_key_store }}" dest="{{ key_stores_location }}/{{ cred_key_store }}" owner="{{ user }}" group="{{ group }}"
+ tags: deploy
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/common/vars/main.yml
----------------------------------------------------------------------
diff --git a/roles/common/vars/main.yml b/roles/common/vars/main.yml
new file mode 100644
index 0000000..0071b42
--- /dev/null
+++ b/roles/common/vars/main.yml
@@ -0,0 +1,5 @@
+---
+# Common variables
+airavata_source_dir: "{{ user_home }}/source"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/gfac/files/mysql-connector-java-5.1.37-bin.jar
----------------------------------------------------------------------
diff --git a/roles/gfac/files/mysql-connector-java-5.1.37-bin.jar b/roles/gfac/files/mysql-connector-java-5.1.37-bin.jar
new file mode 100644
index 0000000..465af67
Binary files /dev/null and b/roles/gfac/files/mysql-connector-java-5.1.37-bin.jar differ
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/gfac/handlers/main.yml
----------------------------------------------------------------------
diff --git a/roles/gfac/handlers/main.yml b/roles/gfac/handlers/main.yml
new file mode 100644
index 0000000..b3d85a9
--- /dev/null
+++ b/roles/gfac/handlers/main.yml
@@ -0,0 +1,8 @@
+---
+
+# Gfac related handlers
+- name: start gfac
+ command: ./bin/airavata-server-start.sh gfac -d chdir="{{ gfac_dir }}/{{ airavata_dist }}/" creates="{{ gfac_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+- name: stop gfac
+ command: ./bin/airavata-server-stop.sh -f chdir="{{ gfac_dir }}/{{ airavata_dist }}/" removes="{{ gfac_dir }}/{{ airavata_dist }}/bin/server_start_*"
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/gfac/tasks/main.yml
----------------------------------------------------------------------
diff --git a/roles/gfac/tasks/main.yml b/roles/gfac/tasks/main.yml
new file mode 100644
index 0000000..bae85cf
--- /dev/null
+++ b/roles/gfac/tasks/main.yml
@@ -0,0 +1,53 @@
+---
+
+################################################################################
+# gfac deployments
+- name: Create Gfac deployment directory
+ file: path="{{ gfac_dir }}" state=directory owner="{{ user }}" group="{{ group }}"
+ when: build|success
+ tags: deploy, update_gfac, update
+
+- name: Check previous deployments
+ stat: path="{{ gfac_dir }}/{{ airavata_dist }}" get_md5=no get_checksum=no
+ register: p
+ tags: deploy, update_gfac, update
+
+- block:
+ - name: stop gfac
+ command: ./bin/airavata-server-stop.sh -f chdir="{{ gfac_dir }}/{{ airavata_dist }}/" removes="{{ gfac_dir }}/{{ airavata_dist }}/bin/server_start_*"
+
+ - name: Delete previous deployments
+ file: path="{{ gfac_dir }}/{{ airavata_dist }}" state=absent
+
+ when: p.stat.exists == True
+ tags: deploy, update_gfac, update
+
+# - block:
+# - name: Remove pervious backup directory
+# file: path="{{ gfac_dir }}/backup" state=absent
+# notify:
+# - stop gfac
+#
+# - name: Backup previous deployment directory
+# copy: src="{{ gfac_dir }}/{{ airavata_dist }}" dest="{{ gfac_dir }}/backup" copy=no
+#
+# when: p.stat.exists == True
+# tags: deploy, update_gfac, update
+
+- name: Copy distribution to gfac deployment directory
+ unarchive: src="{{ airavata_source_dir }}/distribution/target/{{ airavata_dist_name }}" dest="{{ gfac_dir }}/" copy=no
+ become: yes
+ become_user: "{{ user }}"
+ tags: deploy, update_gfac, update
+
+- name: Copy Airavata server properties file
+ template: src=airavata-server.properties.j2 dest="{{ gfac_dir }}/{{ airavata_dist }}/bin/airavata-server.properties" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ tags: deploy, update_gfac, update
+
+- name: Copy Gfac configuration file
+ template: src=gfac-config.yaml.j2 dest="{{ gfac_dir }}/{{ airavata_dist }}/bin/gfac-config.yaml" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ tags: deploy, update_gfac, update
+
+- name: Copy Mysql jar to lib
+ copy: src="{{ mysql_connector_jar }}" dest="{{ gfac_dir }}/{{ airavata_dist }}/lib/{{ mysql_connector_jar }}" owner="{{ user }}" group="{{ group }}"
+ tags: deploy, update_gfac, update
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/gfac/templates/airavata-server.properties.j2
----------------------------------------------------------------------
diff --git a/roles/gfac/templates/airavata-server.properties.j2 b/roles/gfac/templates/airavata-server.properties.j2
new file mode 100644
index 0000000..449a7fd
--- /dev/null
+++ b/roles/gfac/templates/airavata-server.properties.j2
@@ -0,0 +1,278 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+###########################################################################
+#
+# This properties file provides configuration for all Airavata Services:
+# API Server, Registry, Workflow Interpreter, GFac, Orchestrator
+#
+###########################################################################
+
+###########################################################################
+# API Server Registry Configuration
+###########################################################################
+
+#for derby [AiravataJPARegistry]
+#registry.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#registry.jdbc.url=jdbc:derby://localhost:1527/experiment_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+registry.jdbc.driver=com.mysql.jdbc.Driver
+registry.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ exp_catalog }}
+registry.jdbc.user={{ db_user }}
+registry.jdbc.password={{ db_password }}
+#FIXME: Probably the following property should be removed.
+start.derby.server.mode=false
+validationQuery=SELECT 1 from CONFIGURATION
+cache.enable=false
+jpa.cache.size=-1
+#jpa.connection.properties=MaxActive=10,MaxIdle=5,MinIdle=2,MaxWait=60000,testWhileIdle=true,testOnBorrow=true
+
+# Properties for default user mode
+default.registry.user=admin
+default.registry.password=admin
+default.registry.password.hash.method=SHA
+default.registry.gateway=default
+
+###########################################################################
+# Application Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+#appcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#appcatalog.jdbc.url=jdbc:derby://localhost:1527/app_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+appcatalog.jdbc.driver=com.mysql.jdbc.Driver
+appcatalog.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ app_catalog }}
+appcatalog.jdbc.user={{ db_user }}
+appcatalog.jdbc.password={{ db_password }}
+appcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+##########################################################################
+# Replica Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+#replicacatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#replicacatalog.jdbc.url=jdbc:derby://localhost:1527/replica_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+replicacatalog.jdbc.driver=com.mysql.jdbc.Driver
+replicacatalog.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ replica_catalog }}
+replicacatalog.jdbc.user={{ db_user }}
+replicacatalog.jdbc.password={{ db_password }}
+replicacatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+###########################################################################
+# Workflow Catalog DB Configuration
+###########################################################################
+#for derby [AiravataJPARegistry]
+#workflowcatalog.jdbc.driver=org.apache.derby.jdbc.ClientDriver
+#workflowcatalog.jdbc.url=jdbc:derby://localhost:1527/workflow_catalog;create=true;user=airavata;password=airavata
+# MySql database configuration
+workflowcatalog.jdbc.driver=com.mysql.jdbc.Driver
+workflowcatalog.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ replica_catalog }}
+workflowcatalog.jdbc.user={{ db_user }}
+workflowcatalog.jdbc.password={{ db_password }}
+workflowcatalog.validationQuery=SELECT 1 from CONFIGURATION
+
+###########################################################################
+# User Profile MongoDB Configuration
+###########################################################################
+userprofile.mongodb.host=localhost
+userprofile.mongodb.port=27017
+
+
+###########################################################################
+# Server module Configuration
+###########################################################################
+#credential store server should be started before API server
+#This is obsolete property with new script files.
+#servers=credentialstore,apiserver,orchestrator
+
+
+###########################################################################
+# API Server Configurations
+###########################################################################
+apiserver.name={{ api_server_name }}
+apiserver.host={{ api_server_host }}
+apiserver.port={{ api_server_port }}
+apiserver.min.threads=50
+
+###########################################################################
+# API Server SSL Configurations
+###########################################################################
+apiserver.tls.enabled=true
+apiserver.keystore={{ key_stores_location }}/{{ key_store }}
+apiserver.keystore.password=airavata
+
+###########################################################################
+# Orchestrator Server Configurations
+###########################################################################
+orchestrator.server.name={{ orchestrator_name }}
+orchestrator.server.host={{ orchestrator_host }}
+orchestrator.server.port={{ orchestrator_port }}
+orchestrator.server.min.threads=50
+job.validators=org.apache.airavata.orchestrator.core.validator.impl.BatchQueueValidator,org.apache.airavata.orchestrator.core.validator.impl.ExperimentStatusValidator
+submitter.interval=10000
+threadpool.size=10
+start.submitter=true
+embedded.mode=true
+enable.validation=true
+
+
+###########################################################################
+# GFac Server Configurations
+###########################################################################
+gfac.server.name={{ gfac_name }}
+gfac.server.host={{ gfac_host }}
+gfac.server.port={{ gfac_port }}
+gfac.thread.pool.size=50
+host.scheduler=org.apache.airavata.gfac.impl.DefaultHostScheduler
+
+
+
+###########################################################################
+# Airavata Workflow Interpreter Configurations
+###########################################################################
+workflowserver=org.apache.airavata.api.server.WorkflowServer
+enactment.thread.pool.size=10
+
+#to define custom workflow parser user following property
+#workflow.parser=org.apache.airavata.workflow.core.parser.AiravataWorkflowBuilder
+
+
+
+###########################################################################
+# Job Scheduler can send informative email messages to you about the status of your job.
+# Specify a string which consists of either the single character "n" (no mail), or one or more
+# of the characters "a" (send mail when job is aborted), "b" (send mail when job begins),
+# and "e" (send mail when job terminates). The default is "a" if not specified.
+###########################################################################
+
+job.notification.enable=true
+#Provide comma separated email ids as a string if more than one
+job.notification.emailids=
+job.notification.flags=abe
+
+###########################################################################
+# Credential Store module Configuration
+###########################################################################
+credential.store.keystore.url=/home/airavata/airavata_sym.jks
+credential.store.keystore.alias=airavata
+credential.store.keystore.password=airavata
+credential.store.jdbc.url=jdbc:mysql://{{ db_server }}:3306/{{ credential_store }}
+credential.store.jdbc.user={{ db_user }}
+credential.store.jdbc.password={{ db_password }}
+credential.store.jdbc.driver=com.mysql.jdbc.Driver
+credential.store.server.host={{ cred_store_server_host }}
+credential.store.server.port=8960
+credentialstore=org.apache.airavata.credential.store.server.CredentialStoreServer
+
+# these properties used by credential store email notifications
+email.server=smtp.googlemail.com
+email.server.port=465
+email.user=airavata
+email.password=xxx
+email.ssl=true
+email.from=airavata@apache.org
+
+# SSH PKI key pair or ssh password can be used SSH based sshKeyAuthentication is used.
+# if user specify both password sshKeyAuthentication gets the higher preference
+
+################# ---------- For ssh key pair sshKeyAuthentication ------------------- ################
+#ssh.public.key=/path to public key for ssh
+#ssh.private.key=/path to private key file for ssh
+#ssh.keypass=passphrase for the private key
+#ssh.username=username for ssh connection
+## If you set "yes" for ssh.strict.hostKey.checking, then you must provide known hosts file path
+#ssh.strict.hostKey.checking=yes/no
+#ssh.known.hosts.file=/path to known hosts file
+### Incase of password sshKeyAuthentication.
+#ssh.password=Password for ssh connection
+
+################ ---------- BES Properties ------------------- ###############
+#bes.ca.cert.path=<location>/certificates/cacert.pem
+#bes.ca.key.path=<location>/certificates/cakey.pem
+#bes.ca.key.pass=passphrase
+
+###########################################################################
+# Monitoring module Configuration
+###########################################################################
+
+#This will be the primary monitoring tool which runs in airavata, in future there will be multiple monitoring
+#mechanisms and one would be able to start a monitor
+monitors=org.apache.airavata.gfac.monitor.impl.pull.qstat.QstatMonitor,org.apache.airavata.gfac.monitor.impl.LocalJobMonitor
+
+#These properties will used to enable email base monitoring
+email.based.monitor.host=imap.gmail.com
+email.based.monitor.address={{ monitor_email_address }}
+email.based.monitor.password={{ monitor_email_password }}
+email.based.monitor.folder.name=INBOX
+# either imaps or pop3
+email.based.monitor.store.protocol=imaps
+#These property will be used to query the email server periodically. value in milliseconds(ms).
+email.based.monitoring.period=10000
+
+###########################################################################
+# AMQP Notification Configuration
+###########################################################################
+#for simple scenarios we can use the guest user
+#rabbitmq.broker.url=amqp://localhost:5672
+#for production scenarios, give url as amqp://userName:password@hostName:portNumber/virtualHost, create user, virtualhost
+# and give permissions, refer: http://blog.dtzq.com/2012/06/rabbitmq-users-and-virtual-hosts.html
+rabbitmq.broker.url={{ rabbitmq_broker_url }}
+status.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
+task.launch.publisher=org.apache.airavata.messaging.core.impl.RabbitMQProcessLaunchPublisher
+rabbitmq.status.exchange.name=airavata_rabbitmq_exchange
+rabbitmq.task.launch.exchange.name=airavata_task_launch_rabbitmq_exchange
+durable.queue=false
+prefetch.count=200
+launch.queue.name=launch.queue
+cancel.queue.name=cancel.queue
+activity.publisher=org.apache.airavata.messaging.core.impl.RabbitMQStatusPublisher
+rabbitmq.exchange.name=airavata_rabbitmq_exchange
+
+###########################################################################
+# Zookeeper Server Configuration
+###########################################################################
+embedded.zk=false
+zookeeper.server.connection=localhost:2181
+zookeeper.timeout=30000
+
+########################################################################
+## API Security Configuration
+########################################################################
+api.secured=true
+security.manager.class=org.apache.airavata.api.server.security.DefaultAiravataSecurityManager
+### TLS related configuration ####
+TLS.enabled=true
+TLS.api.server.port=9930
+TLS.client.timeout=10000
+#### keystore configuration ####
+keystore.path={{ key_stores_location }}/{{ key_store }}
+keystore.password=airavata
+#### trust store configuration ####
+trust.store={{ key_stores_location }}/{{ cred_key_store }}
+trust.store.password=airavata
+#### remote authorization server url ####
+remote.oauth.authorization.server=https://idp.scigap.org:9443/services/
+#### xacml based authorization policy ####
+authorization.policy=airavata-default-xacml-policy
+#### authorization cache related configuration ####
+authz.cache.enabled=true
+authz.cache.manager.class=org.apache.airavata.api.server.security.authzcache.DefaultAuthzCacheManager
+in.memory.cache.size=1000
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/gfac/templates/gfac-config.yaml.j2
----------------------------------------------------------------------
diff --git a/roles/gfac/templates/gfac-config.yaml.j2 b/roles/gfac/templates/gfac-config.yaml.j2
new file mode 100644
index 0000000..3df5832
--- /dev/null
+++ b/roles/gfac/templates/gfac-config.yaml.j2
@@ -0,0 +1,111 @@
+##################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+################################################################
+
+jobSubmitters:
+ - submissionProtocol: SSH
+ taskClass: org.apache.airavata.gfac.impl.task.DefaultJobSubmissionTask
+# properties:
+# - userName: airavata
+# passPhrase: airavata
+# privateKey: /path/to/the/privatekey
+# publicKey: /path/to/the/publickey
+# hostName: remote.client.hostName
+
+ - submissionProtocol: SSH_FORK
+ taskClass: org.apache.airavata.gfac.impl.task.ForkJobSubmissionTask
+
+ - submissionProtocol: LOCAL
+ taskClass: org.apache.airavata.gfac.impl.task.DefaultJobSubmissionTask
+
+# Following job subbmitters are not yet implemented.
+
+ # - submissionProtocol: GSISSH
+ # taskClass: org.apache.airavata.task.adapters.GSISSHJobSubmissionTask
+commonTasks:
+ - type: SETUP
+ taskClass: org.apache.airavata.task.common.SetupTask
+
+ - type: CLEANUP
+ taskClass: org.apache.airavata.task.common.CleanupTask
+
+fileTransferTasks:
+# - transferProtocol: SCP
+# taskClass: org.apache.airavata.gfac.impl.task.DataStageTask
+
+# If your client doen't run the same instance where airavata server is running then you need to comment above
+# DataStageTask and uncomment SCPDataStageTask. To work with SCPDataStageTask, you either need to
+# provide ssh keys or password.
+
+ - transferProtocol: SCP
+ taskClass: org.apache.airavata.gfac.impl.task.SCPDataStageTask
+ properties:
+ - userName: root
+ passPhrase: ultrascan
+ privateKeyPath: /home/airavata/.ssh/id_rsa
+ publicKeyPath: /home/airavata/.ssh/id_rsa.pub
+ hostName: gw75.iu.xsede.org
+ inputPath: /var/www/portal/experimentData/
+# password: password
+
+# Following transfer tasks are not yet implemented.
+ #- transferProtocol: SFTP
+ # taskClass: org.apache.airavata.task.adapters.SFTPFileTransferTask
+
+ #- transferProtocol: GRIDFTP
+ # taskClass: org.apache.airavata.task.adapters.GRIDFTPFileTransferTask
+
+ #- transferProtocol: LOCAL
+ # taskClass: org.apache.airavata.task.adapters.LocalFileTransferTask
+
+resources:
+ - jobManagerType: PBS
+ commandOutputParser: org.apache.airavata.gfac.impl.job.PBSOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.PBSEmailParser
+ resourceEmailAddresses:
+ - pbsconsult@sdsc.edu # gordon
+ - adm@trident.bigred2.uits.iu.edu # Bigred2
+ - root <ad...@trident.bigred2.uits.iu.edu> # Bigred2
+ - root <ad...@scyld.localdomain> # alamo
+ - root <ad...@m2.karst.uits.iu.edu> #karst
+
+ - jobManagerType: SLURM
+ commandOutputParser: org.apache.airavata.gfac.impl.job.SlurmOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.SLURMEmailParser
+ resourceEmailAddresses:
+ - SDSC Admin <sl...@comet-fe3.sdsc.edu> # comet
+ - slurm@batch1.stampede.tacc.utexas.edu # stampede
+ - SDSC Admin <sl...@comet-fe4.sdsc.edu> # comet new
+ - Slurm <sl...@psc.edu> # bridges
+ - Slurm Daemon <sl...@odin.oscer.ou.edu> # OU Schooner
+
+ - jobManagerType: UGE
+ commandOutputParser: org.apache.airavata.gfac.impl.job.UGEOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.UGEEmailParser
+ resourceEmailAddresses:
+ - ls4.tacc.utexas.edu # contain Lonestar
+
+ - jobManagerType: LSF
+ commandOutputParser: org.apache.airavata.gfac.impl.job.LSFOutputParser
+ emailParser: org.apache.airavata.gfac.monitor.email.parser.LSFEmailParser
+ resourceEmailAddresses:
+ - iu.xsede.edu # test resource mail address
+
+ - jobManagerType: FORK
+ commandOutputParser: org.apache.airavata.gfac.impl.job.ForkOutputParser
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/gfac/vars/main.yml
----------------------------------------------------------------------
diff --git a/roles/gfac/vars/main.yml b/roles/gfac/vars/main.yml
new file mode 100644
index 0000000..f5451c3
--- /dev/null
+++ b/roles/gfac/vars/main.yml
@@ -0,0 +1,3 @@
+---
+# Gfac related variables
+gfac_dir: "{{ deployment_dir }}/gfac-instance"
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/rabbitmq/handlers/main.yml
----------------------------------------------------------------------
diff --git a/roles/rabbitmq/handlers/main.yml b/roles/rabbitmq/handlers/main.yml
new file mode 100644
index 0000000..4515589
--- /dev/null
+++ b/roles/rabbitmq/handlers/main.yml
@@ -0,0 +1,10 @@
+---
+# Rabbitmq related handlers
+- name: start rabbitmq
+ service: name=rabbitmq-server state=started
+
+- name: stop rabbitmq
+ service: name=rabbitmq-server state=stopped
+
+- name: restart rabbitmq
+ service: name=rabbitmq-server state=restarted
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/rabbitmq/tasks/main.yml
----------------------------------------------------------------------
diff --git a/roles/rabbitmq/tasks/main.yml b/roles/rabbitmq/tasks/main.yml
new file mode 100644
index 0000000..1d0647b
--- /dev/null
+++ b/roles/rabbitmq/tasks/main.yml
@@ -0,0 +1,44 @@
+---
+
+
+################################################################################
+# Setup and run rabbitmq
+- name: Install erlang latest version
+ yum: name=https://www.rabbitmq.com/releases/erlang/erlang-18.3-1.el7.centos.x86_64.rpm state=present
+ tags: rabbitmq, dependency
+
+
+- name: Install Rabbitmq rpm
+ yum: name=https://www.rabbitmq.com/releases/rabbitmq-server/v3.6.3/rabbitmq-server-3.6.3-1.noarch.rpm state=present
+ tags: rabbimq, dependency
+
+ # add hostname to /etc/hosts file
+- name: Edit /etc/hosts file
+ lineinfile: dest=/etc/hosts line="{{ host_address }} {{host_name}}"
+ notify:
+ - restart rabbitmq
+ tags: rabbitmq, dependency
+
+- name: Start Rabbitmq server
+ service: name=rabbitmq-server state=started
+ tags: rabbitmq, dependency
+
+- name: Enable rabbitmq management plugin
+ rabbitmq_plugin: names=rabbitmq_management state=enabled
+ tags: rabbitmq, dependency
+
+- name: Create rabbitmq vhost {{ rabbitmq_vhost }}
+ rabbitmq_vhost: name="{{ rabbitmq_vhost }}" state=present
+ tags: rabbitmq, dependency
+
+- name: Add user {{ rabbitmq_user }} to vhost {{ rabbitmq_vhost }} and give permission
+ rabbitmq_user: user="{{ rabbitmq_user }}"
+ password="{{ rabbitmq_password }}"
+ vhost="{{ rabbitmq_vhost }}"
+ configure_priv=.*
+ read_priv=.*
+ write_priv=.*
+ state=present
+ tags: rabbitmq, dependency
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/setup/tasks/main.yml
----------------------------------------------------------------------
diff --git a/roles/setup/tasks/main.yml b/roles/setup/tasks/main.yml
new file mode 100644
index 0000000..a20b018
--- /dev/null
+++ b/roles/setup/tasks/main.yml
@@ -0,0 +1,44 @@
+---
+#Tasks file can include smaller files if wanted
+#All commons tasks goes here
+- name: Create a new user group "{{ group }}"
+ group: name={{ group }}
+ tags: user
+
+- name: Create a new user "{{ user }}"
+ user: name={{ user }} group={{ group }}
+ tags: user
+
+################################################################################
+- name: Install git latest version
+ yum: name=git state=latest update_cache=yes
+ tags: env
+
+- name: Install maven latest version
+ yum: name=maven state=latest update_cache=yes
+ tags: env
+
+################################################################################
+# Install Orcal Java
+- name: download oracle java 8 rpm
+ get_url: url="{{ java_rpm_url }}" dest="{{ java_dir_source }}" headers='Cookie:oraclelicense=accept-securebackup-cookie'
+ tags: env
+
+- name: Install oracle java 8
+ yum: name="{{ java_dir_source }}/{{ java_rpm_filename }}" state=present
+ tags: env
+
+- name: set Java version as default
+ alternatives:
+ name="{{ item.exe }}"
+ link="/usr/bin/{{ item.exe }}"
+ path="{{ item.path }}/{{ item.exe }}"
+ with_items:
+ - { path: "{{ java_home }}/jre/bin", exe: 'java' }
+ - { path: "{{ java_home }}/jre/bin", exe: 'keytool' }
+ - { path: "{{ java_home }}/bin", exe: 'javac' }
+ - { path: "{{ java_home }}/bin", exe: 'javadoc' }
+ tags: env
+
+# End
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/setup/tasks/redhat.yml
----------------------------------------------------------------------
diff --git a/roles/setup/tasks/redhat.yml b/roles/setup/tasks/redhat.yml
new file mode 100644
index 0000000..a0e56c7
--- /dev/null
+++ b/roles/setup/tasks/redhat.yml
@@ -0,0 +1,12 @@
+---
+- name: Install git latest version
+ yum: name=git state=latest update_cache=yes
+# become: true
+# become_user: airavata
+ tags: env_setup
+
+- name: Install maven latest version
+ yum: name=maven state=latest update_cache=yes
+ tags: env_setup
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/setup/vars/main.yml
----------------------------------------------------------------------
diff --git a/roles/setup/vars/main.yml b/roles/setup/vars/main.yml
new file mode 100644
index 0000000..a995684
--- /dev/null
+++ b/roles/setup/vars/main.yml
@@ -0,0 +1,15 @@
+---
+#Variables associated with this role
+# Oracle Java 8
+java_dir_source: "/usr/local/src"
+
+java_version: 8
+java_version_update: 91
+java_version_build: '14'
+java_version_string: "1.{{ java_version }}.0_{{ java_version_update }}"
+java_home: "/usr/java/jdk1.{{ java_version }}.0_{{ java_version_update }}"
+
+java_rpm_filename: "jdk-{{ java_version }}u{{ java_version_update }}-linux-x64.rpm"
+java_rpm_url: "http://download.oracle.com/otn-pub/java/jdk/{{ java_version }}u{{ java_version_update }}-b{{ java_version_build }}/{{ java_rpm_filename }}"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/zookeeper/handlers/main.yml
----------------------------------------------------------------------
diff --git a/roles/zookeeper/handlers/main.yml b/roles/zookeeper/handlers/main.yml
new file mode 100644
index 0000000..c6d7c34
--- /dev/null
+++ b/roles/zookeeper/handlers/main.yml
@@ -0,0 +1,19 @@
+---
+
+# Zookeeper related handlers
+- name: start zookeeper
+ command: ./bin/zkServer.sh start chdir={{zookeeper_dir}}/ creates={{zookeeper_data_dir}}/zookeeper_server.pid
+ become: true
+ become_user: "{{ user }}"
+
+- name: stop zookeeper
+ command: ./bin/zkServer.sh stop chdir={{zookeeper_dir}}/ removes={{zookeeper_data_dir}}/zookeeper_server.pid
+ become: true
+ become_user: "{{ user }}"
+
+- name: restart zookeeper
+ command: ./bin/zkServer.sh restart chdir={{zookeeper_dir}}/ removes={{zookeeper_data_dir}}/zookeeper_server.pid
+ become: true
+ become_user: "{{ user }}"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/zookeeper/tasks/main.yml
----------------------------------------------------------------------
diff --git a/roles/zookeeper/tasks/main.yml b/roles/zookeeper/tasks/main.yml
new file mode 100644
index 0000000..c1fd86b
--- /dev/null
+++ b/roles/zookeeper/tasks/main.yml
@@ -0,0 +1,15 @@
+---
+################################################################################
+# Setup and run Zookeeper
+- name: Download and unarchive zookeeper
+ unarchive: src="{{ zookeeper_url }}" dest="{{ user_home }}" copy=no owner="{{ user }}" group="{{ group }}"
+ tags: zookeeper, dependency
+
+- name: Copy zoo.cfg file
+ template: src=zoo.cfg.j2 dest="{{ zookeeper_dir }}/conf/zoo.cfg" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ notify:
+ - start zookeeper
+ - restart zookeeper
+ tags: zookeeper, dependency
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/zookeeper/templates/zoo.cfg.j2
----------------------------------------------------------------------
diff --git a/roles/zookeeper/templates/zoo.cfg.j2 b/roles/zookeeper/templates/zoo.cfg.j2
new file mode 100644
index 0000000..8426b98
--- /dev/null
+++ b/roles/zookeeper/templates/zoo.cfg.j2
@@ -0,0 +1,28 @@
+# The number of milliseconds of each tick
+tickTime={{tick_time}}
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit={{init_limit}}
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit={{sync_limit}}
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just
+# example sakes.
+dataDir={{zookeeper_data_dir}}
+# the port at which the clients will connect
+clientPort={{ client_port }}
+# the maximum number of client connections.
+# increase this if you need to handle more clients
+#maxClientCnxns=60
+#
+# Be sure to read the maintenance section of the
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+#autopurge.purgeInterval=1
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/roles/zookeeper/vars/main.yml
----------------------------------------------------------------------
diff --git a/roles/zookeeper/vars/main.yml b/roles/zookeeper/vars/main.yml
new file mode 100644
index 0000000..dd40e56
--- /dev/null
+++ b/roles/zookeeper/vars/main.yml
@@ -0,0 +1,17 @@
+---
+#Variables associated with this role
+# zookeeper related variable
+zookeeper_version: 3.4.8
+zookeeper_url: http://www.us.apache.org/dist/zookeeper/zookeeper-{{zookeeper_version}}/zookeeper-{{zookeeper_version}}.tar.gz
+
+apt_cache_timeout: 3600
+client_port: 2183
+init_limit: 5
+sync_limit: 2
+tick_time: 2000
+data_dir: /var/lib/zookeeper
+log_dir: /var/log/zookeeper
+zookeeper_dir: "{{ user_home }}/zookeeper-{{zookeeper_version}}"
+zookeeper_data_dir: "{{ zookeeper_dir }}/data"
+
+...
http://git-wip-us.apache.org/repos/asf/airavata/blob/56384a1a/site.yml
----------------------------------------------------------------------
diff --git a/site.yml b/site.yml
new file mode 100644
index 0000000..cf71290
--- /dev/null
+++ b/site.yml
@@ -0,0 +1,25 @@
+---
+#Master playbook
+- hosts: zookeeper
+ roles:
+ - setup
+ - zookeeper
+
+- hosts: rabbitmq
+ roles:
+ - setup
+ - rabbitmq
+
+- hosts: gfac
+ roles:
+ - setup
+ - common
+ - gfac
+
+- hosts: api-orch
+ roles:
+ - setup
+ - common
+ - api-orch
+
+...