You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/07/15 14:23:19 UTC
[airavata-data-lake] branch master updated: Finalizing ansibles for
custos, mft and datalke deployment
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 2d9633f Finalizing ansibles for custos, mft and datalke deployment
2d9633f is described below
commit 2d9633f71c16bd03b7dd4272a812d7d95b0506d3
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Thu Jul 15 10:23:07 2021 -0400
Finalizing ansibles for custos, mft and datalke deployment
---
ansible/README.md | 2 +-
ansible/datalake.yml | 4 +-
ansible/inventories/test/group_vars/all/vars.yml | 36 ++++-
ansible/inventories/test/group_vars/all/vault.yml | 150 +++++++++++++++++++--
ansible/roles/custos/tasks/main.yml | 26 ++++
ansible/roles/data_lake/tasks/main.yml | 59 +++++++-
.../custos-data-synchronizer/config.yml.j2 | 17 +++
.../data-orchestrator/application.properties.j2 | 21 +++
.../templates/data-orchestrator/config.yml.j2 | 17 +++
.../templates/drms}/application.properties.j2 | 25 ++--
ansible/roles/mft/tasks/main.yml | 32 ++---
.../mft/templates/agent/application.properties.j2 | 1 +
.../mft/templates/secret-service/secrets.json.j2 | 11 +-
13 files changed, 346 insertions(+), 55 deletions(-)
diff --git a/ansible/README.md b/ansible/README.md
index 1de3347..21a51bf 100644
--- a/ansible/README.md
+++ b/ansible/README.md
@@ -26,4 +26,4 @@ Now you should be ready to run `ansible-playbook` and other ansible commands.
## Useful commands
-- Deploy Data Lake: `ansible-playbook -i inventories/path/to/inventory/dir datalake.yml`
\ No newline at end of file
+- Deploy Data Lake: `ansible-playbook -i inventories/{inventory}/ --vault-password-file=vault-password.txt datalake.yml`
\ No newline at end of file
diff --git a/ansible/datalake.yml b/ansible/datalake.yml
index 0673e59..70c4bae 100644
--- a/ansible/datalake.yml
+++ b/ansible/datalake.yml
@@ -21,4 +21,6 @@
---
- hosts: data-lake
roles:
- - mft
\ No newline at end of file
+ - custos
+ - mft
+ - data_lake
\ No newline at end of file
diff --git a/ansible/inventories/test/group_vars/all/vars.yml b/ansible/inventories/test/group_vars/all/vars.yml
index 7ed3c10..3e18c43 100644
--- a/ansible/inventories/test/group_vars/all/vars.yml
+++ b/ansible/inventories/test/group_vars/all/vars.yml
@@ -37,10 +37,16 @@ mft_repo: "https://github.com/apache/airavata-mft.git"
mft_git_branch: develop
mft_distribution_version: 0.01
+custos_source_dir: "{{ user_home }}/custos-source"
+custos_repo: "https://github.com/apache/airavata-custos.git"
+custos_git_branch: develop
+
mft_default_agent_id: agent0
-mft_default_agent_host: 149.165.157.235
+mft_default_agent_host: 10.1.0.33
+mft_default_agent_advertised_host: 149.165.157.235
mft_default_agent_port: 3333
+mft_api_service_host: localhost
mft_api_service_http_port: 8088
mft_api_service_grpc_port: 7004
mft_api_service_id: 0
@@ -65,5 +71,31 @@ consul_install_dir: /usr/local/bin
consul_config_dir: /etc/consul
consul_data_dir: /var/data
+datalake_distribution_version: 0.01-SNAPSHOT
datalake_drms_host: localhost
-datalake_drms_grpc_port: 7070
\ No newline at end of file
+datalake_drms_grpc_port: 7070
+
+datalake_drms_neo4j_uri: bolt://149.165.156.173:7687
+datalake_drms_neo4j_user: "{{ vault_datalake_drms_neo4j_user }}"
+datalake_drms_neo4j_password: "{{ vault_datalake_drms_neo4j_password }}"
+datalake_drms_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
+datalake_drms_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"
+datalake_drms_custos_host: custos.scigap.org
+datalake_drms_custos_port: 31499
+datalake_drms_http_port: 8900
+
+datalake_synch_broker_url: 149.165.156.200:9092
+datalake_synch_broker_consumer_group: custosEventsGroup
+datalake_synch_broker_topic: 10002708-754b89b5-3a57-496c-aa34-8e2b4916fbcc
+datalake_synch_tenant_id: custos-whedmgamitu357p4wuke-10002708
+
+datalake_data_orch_datasource_url: jdbc:mysql://localhost:3306/data_orchestrator?useSSL=false&serverTimezone=UTC&useLegacyDatetimeCode=false&createDatabaseIfNotExist=true&allowPublicKeyRetrieval=true
+datalake_data_orch_datasource_username: "{{ vault_datalake_data_orch_datasource_username }}"
+datalake_data_orch_datasource_password: "{{ vault_datalake_data_orch_datasource_password }}"
+datalake_data_orch_http_port: 8899
+datalake_data_orch_broker_url: localhost:9092
+datalake_data_orch_broker_consumer_group: DataOrchestratorGroup3
+datalake_data_orch_broker_topic: data-orchestrator-file-events
+
+datalake_workflow_engine_host: localhost
+datalake_workflow_engine_port: 6565
\ No newline at end of file
diff --git a/ansible/inventories/test/group_vars/all/vault.yml b/ansible/inventories/test/group_vars/all/vault.yml
index bd5ff07..75955ac 100644
--- a/ansible/inventories/test/group_vars/all/vault.yml
+++ b/ansible/inventories/test/group_vars/all/vault.yml
@@ -1,14 +1,138 @@
$ANSIBLE_VAULT;1.1;AES256
-65656438633938663065663236323133366632333430643938643039316430383463313534396666
-3436653732343937656161356661373230336238386635340a333635343033666363303265393666
-30633336333035373030383762656661303964356433653563373066613939333663636661643862
-6430313933353633620a306162613765376461613064303163383934636337623964343634316163
-62613134386265343666626364353331616364383735353333633538366566663635316662353434
-62623234336638343265326437323266646238663437323436303836666130663533663266356635
-64313330303762623434316636383965396338616361646539633866636633393536366666326662
-63613830353839346437333737313038373736653036383638353937393031666630653664343536
-38353464353564623437623632613966333738396566363239336166636466333833396261386230
-37623233363131626631393832383361643030663661656165393261653836366131326564643037
-36643234373833346632613366313262663434313937383665613734636631613934343961633561
-32663666303431336664343463656237336332393264613132323266633765316532373434636264
-3536
+34346139356162346462663236313166653634333032656438313330613439653739386161383530
+3038326162353463316537393532636162643464303266310a306565613164616235363832613164
+63346130323465666332313330653562306630383633646162626666323236643166353831656531
+6337383936613535350a633037343835343966636366656532626339613566666466656633646464
+66666237613864626664363863626634366266626238336561366139376332616136393065393332
+31393136613939303264313231666534633230343062646634373765313531363335636665353761
+31373665643532333732376334306163353664373333656263333761656563393534383766633238
+62343534353839643862316662613237383634633232373461303336346337363735626531646636
+63653264626334613831323961613064383034363131373638373534393036366236666131326663
+35373538653161373238663030636135616635666630663434613466363938303565663536326466
+36346561326137313030316434303563316132396539653838396538626661656335393738363932
+37306630386564313239626332363539636363306139363934333364616465623335643537633661
+63663361623336353832383132333837386234383537383234323838326463373030646235316137
+34343731643131613361613337383937353235306539613632653337623936303566626365393139
+30636635396333633338383261383031393936663036623235396462353232396135396665303631
+61343365353433346233313536333430653463366630386565343862653331636537613663396534
+31343137646564633864353861316130396139633163366562663633326138353863373735313335
+37373239636335626430333563646637313961623434303135623161323638323037383033316230
+36656537336637346631353963343937336230316232376262623232323766396232303565396338
+61306465336365393935386533636237336234613861376430646136333863346363666130313336
+39333064656265343431316661376434663033393531383934333162333362326434306365353464
+38663137616333326431346465353265303365653730393633633165313034363233666238666362
+37383832646632326438643061346161663236326666306233363863356662663562373830336539
+33353839633865363865326330636534646131306639333164383132383366313333333263386336
+39336434376163366333663432653732623337613235633565353264393632376262313262313661
+35343231323334363362343238613938346436663332343665386531333538373435353537343361
+36393264383166363136623139633466396138363436353534653662343239653065663761663734
+61616662396265356239613462643664623461323133626239636366313735633863393962323765
+66376361363033333031303766343834373137363234626635636230326237363465316163383831
+36616136303262306263623862303937353939396236356530363631636166613762316434666135
+65613438306362346437353039303562343837396132613333636534396262636337306463363264
+65616235633761333962373032363430353466313733336666336266343339383663613535353535
+39333166316133646137323338336264383835613962396434653430323738613334663833376666
+38623062656539326332386334626630323837326363353766336434376633376131636561326162
+64363732343835386231653934623434323062643534306562643036626135323237663065373662
+34636365633632346537313538626632623431346330656638323233666237613664656133386464
+31326261316236613764623030663039383934343630653231656630376239343634363738623134
+33316532376561353134383836386664376465386435613632353935626632653036323561613530
+62313834613965636161383366653234353563333964616331393962653431323662626663333666
+65366462303465653365326665363138333965643233346333613933633964343863316638656633
+32393936646666613366623936613164303433376339373336663463633566376161353663333931
+34346335643864383261386630656331326638386334333935656364613830396564633535306562
+39303262643530663238376163323931666462333865656432323831623762383639383131653338
+33626530616466396434626264623637316135323035336664633465326462366434316266393331
+61346339333236633931373463386266636230393332343535363463323537663931326662356130
+32373264313633336564303633383430656432373034383038333434643431353937663862656532
+32313264396234643738383233373433616336393664633431656438363130316439373833396132
+62376234636635376266643464643531613339616636656131616238623531663637373839356635
+30653234353466666233316164326430303230373134663339326664633061656633356437373635
+63623865316639343938666439313063396636623237353134303936383465303132336336666361
+64313431656163663462393631356334656666396235346366613662313231633264386338613536
+37636465316164336162613862633837353938656563373037353438393666376532303530663332
+66306639366337373839393336613331663036613737306336313130383563626335656431653461
+64623863363930343963316333363236356661343130646230633238333465366361313666633932
+64386234626165316137333138653562373430323032376439386330303664396662323238626264
+33343666353864656138373335373566386237633431636237313366366161373366363265366639
+34323230336638636662633638653232386164313730323735356130626431643061623733356231
+63666438633032626533333531313430613436393566613735353436346231663765373233333337
+61613636303566323565363731663733313463656134643734303261626365613237653439343131
+63313136663531353430663466663466343664663566643735393131623536643761643831303139
+35313364613463623563383432623832346439386237633764646138633831616531363034666566
+31313639326662633933373966663663616566393239316330626139343062616233323264333437
+63303834636630333939636265333363343237343861643737303330393161333936323334613736
+34343631383732363864623366376431363637383130303636383364363936643561613230316563
+63663831653965636632363034363231373265303466366434383337386433613864363364663532
+39373036386138393461346363366536313439663437656538363166393935313766353663303764
+30396464356335653135626638336336363065396464663935393634666638646566636163353937
+38626262353738646431356637633839636234373934303566326336383633336633656361633837
+36336365323362316564343336333734363532323463313433383134343332353633346330343465
+35316637343334643331393530396132613231646635613634383561653436383561313736646566
+64346366306633623632336235653537393963356531363238313436636135353338386563666562
+36383635313734326265396533346631313634653336366537333430313933633336306633333966
+33656563386436656666623435656433353563373163363037393565383935353665396631643338
+62386133336637373162333566643965343637393633613230353739633139633238383066373066
+37366332616163363237636639303962663431366636646666646339323738643437666639383836
+66626331363239386436363037363832626335376465343431356263306238313635313362313737
+61396665323834613034393664316366313535383831663564346164613530313335636261353565
+66323961326332666466643432313230383231353637353538386163643835356662643663336532
+33386566373763323662363635313037633964623135656566323964366161356435386533393135
+31363637616661366361636636656263323232333031353735343761343232303163316261663035
+31356263396538316534366638666462313864666435623462646437616131363661323237623730
+32316333393564363832326335653637343839653838393332626264393832613833336635326336
+33313938626236373066666131373332636331666362653035346633323865363235323631653366
+32373361646333306337623363643630643866346561323463616233383436363033343866383831
+62633063613764333436323161386338666334356562333863303863376265303937303961323238
+37316366393332666164626335373361636230313937353363396538313662366330316235316339
+37363863353538333934656234663533623531393162333239643862623734326530376362303665
+33323934326130656131633065343630343066663762643439363464616139616134343939393237
+39306562623864303961373866636565626539393266616362353437356537353362633830303064
+36373730653831363230376632313530326461663935643233666338326633633838333231363562
+31326664633762633139343632656462653834376630313139383437346666363531373736666134
+34613265633262643235323439363661663338663535633039383466363439653039666338393062
+39653233633738376265333263323635343437353239353838393139313933663261353533333663
+30663438663139306134656632396534613666663838356438386566343138383436626434663364
+65313136613862313361306464373037383837303437626663333062643864343831613261336430
+38613038366331623862636561343762623361626661636431353938396637343336386366376431
+35376165316237333831323232623564373563623233616236353065343665616433643434356230
+63343461396130653132663137343134666531663630643032346632323435346332623266353638
+31323434343931376366396166633965306661363231343034333335663831373131316433376636
+39343266326261626138646664666232306630626132343738343938356266326336313839393735
+65323936626666666636346364653966646162336566356663333763633064663431366463663232
+61663666336333363736343432626434303765663834353833323962653562316634636665353765
+38656336623735396236356234623330383832363866666231363334636366636666393762343332
+39303938623739386164653863653938613639323337346632376364383936366665623261373438
+63333035636336356135656132353037323936323832633437363938326436623832323837393562
+38366234393734366638303333623932323432633633386234613637376361633334633962646561
+65656339323237363834616364303063383532363738386133393562633335373830343366623836
+62393262373637363737376166643931386663363836646533643239323434353932363035656136
+30363361653730356531333835393333376132306534353763323933303037383733303664303331
+62393430623630623332623965653661653830316638626433326362623062613564376265373539
+35373065366133363436336134333837663262666138646436373134656364656235376465383766
+66666263333230323238336538356563353332303433323561663732353261336431323836656136
+35303866613537643931353737383030653064626333396365393338316161373231383861356437
+39306434636135393065346165346535616163643537616137366132343830396664366365646433
+63336134306566363764656330626534646632343534333263623534313031616236613866643835
+32333533663364643831663735663236303936343263653163303039633334656430313536363732
+38636333366136393336386239356231303434396539633738323863626234653735373533393632
+37396337383761613564396433613461613030373961656236643130383033393134363635346363
+63326462313465633637316337396662643634363236366166643863323031666436663265356464
+35393535303032376430386362623963643263323230616139336638383735333935326638643665
+33353664393430633638633833303564323463656535323363366233396233353661663265653239
+39653562313465623838613661386265626134636462666662663032656436306435633138626238
+32313634313533313536613139613962613937643639633937356464633733396634666665363165
+62346261386533663365373464613863393263353337666162666432303835386334313932363761
+32383763633666306261626336636231373234623538633238346235326636626336613365316261
+66313666653664383834306630383734333937363336393364666237656162376264383730623861
+65633136373537623735613262326231666631393338343562626462306164356532366338633330
+36343534316361613661373935643736373965303532636538663862663966643466613436656634
+65386361303532346462336564316231303939623262616133306633356462386362346237303233
+63306335393862663137323236343936366238303139373931383035303862323563656535656333
+34633765336431383162646134656236656331306461343236326466656239353161343237393432
+61656662666430376637666331353030656664386565316237346236623236373836353939633531
+34386465393066666262333263633136316136626537613235613363363332336232386633653130
+65333362343334613464663431663533353265633730333635616261636562353363666563303765
+39656337623665343766636436383831343930616664626462313936323934376434383463356139
+63653763616637353932353130393031393237646337396362656562353339383037666564323630
+63313964613133356531303561363337393866346138336562643861643162313736
diff --git a/ansible/roles/custos/tasks/main.yml b/ansible/roles/custos/tasks/main.yml
new file mode 100644
index 0000000..2122aee
--- /dev/null
+++ b/ansible/roles/custos/tasks/main.yml
@@ -0,0 +1,26 @@
+- name: Create Custos source directory
+ become: yes
+ file: path={{ custos_source_dir }}
+ state=directory
+ mode=0755
+ owner={{ user }}
+ group={{ group }}
+
+
+- name: git checkout from Custos github repo {{ custos_repo }} branch {{ custos_git_branch }}
+ git: repo="{{ custos_repo }}"
+ dest="{{ custos_source_dir }}"
+ version="{{ custos_git_branch }}"
+ register: checkout
+ tags: update
+ become: yes
+ become_user: "{{ user }}"
+
+- name: Run Custos maven build
+ command: mvn clean install -Dmaven.test.skip=true chdir="{{ custos_source_dir }}/"
+ environment:
+ MAVEN_OPTS: "-Xmx2048m"
+ register: build
+ tags: update
+ become: yes
+ become_user: "{{ user }}"
\ No newline at end of file
diff --git a/ansible/roles/data_lake/tasks/main.yml b/ansible/roles/data_lake/tasks/main.yml
index 69943eb..1e4c224 100644
--- a/ansible/roles/data_lake/tasks/main.yml
+++ b/ansible/roles/data_lake/tasks/main.yml
@@ -28,6 +28,11 @@
zone=public permanent=true state=enabled immediate=yes
become: yes
+- name: open firewall port {{ datalake_data_orch_http_port }} for Data Orchestrator HTTP connections
+ firewalld: port="{{ datalake_data_orch_http_port }}/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
- name: Create Datalake deployment directory {{ datalake_deployment_dir }}
become: yes
file: path={{ datalake_deployment_dir }}
@@ -38,7 +43,7 @@
- name: Create Datalake source directory
become: yes
- file: path={{datalake_source_dir}}
+ file: path={{ datalake_source_dir }}
state=directory
mode=0755
owner={{ user }}
@@ -62,3 +67,55 @@
become: yes
become_user: "{{ user }}"
+- name: Stop daemons if running
+ command: "{{ item.command }} chdir={{ datalake_deployment_dir }}/{{ item.dir }}/"
+ with_items:
+ - { command: ./bin/drms-daemon.sh stop, dir: "DRMS-{{datalake_distribution_version}}" }
+ - { command: ./bin/drms-custos-synchronizer-daemon.sh stop, dir: "DRMS-Custos-Synchronizer-{{datalake_distribution_version}}" }
+ - { command: ./bin/orch-api-server-daemon.sh stop, dir: "Orchestrator-API-Service-{{datalake_distribution_version}}" }
+ ignore_errors: yes
+ become: yes
+ become_user: "{{ user }}"
+
+- name: Copy Datalake distributions to Datalake deployment directory
+ unarchive: "src={{ datalake_source_dir }}/{{ item }}
+ dest={{ datalake_deployment_dir }}/ copy=no"
+ with_items:
+ - "data-resource-management-service/drms-api/target/DRMS-{{datalake_distribution_version}}-bin.zip"
+ - "data-resource-management-service/drms-custos-synchronizer/target/DRMS-Custos-Synchronizer-{{datalake_distribution_version}}-bin.zip"
+ - "data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/target/Orchestrator-API-Service-{{datalake_distribution_version}}-bin.zip"
+ become: yes
+ become_user: "{{ user }}"
+
+- name: Copy Datalake property files
+ template: "src={{ item.name }}
+ dest={{ datalake_deployment_dir }}/{{ item.dir }}/conf/{{ item.target }}
+ owner={{ user }}
+ group={{ group }}
+ mode=\"u=rw,g=r,o=r\""
+ with_items:
+ - { name: drms/application.properties.j2,
+ dir: "DRMS-{{datalake_distribution_version}}",
+ target: application.properties}
+ - { name: custos-data-synchronizer/config.yml.j2,
+ dir: "DRMS-Custos-Synchronizer-{{datalake_distribution_version}}",
+ target: config.yml }
+ - { name: data-orchestrator/application.properties.j2,
+ dir: "Orchestrator-API-Service-{{datalake_distribution_version}}",
+ target: application.properties }
+ - { name: data-orchestrator/config.yml.j2,
+ dir: "Orchestrator-API-Service-{{datalake_distribution_version}}",
+ target: config.yml }
+ become: yes
+ become_user: "{{ user }}"
+
+
+- name: Start Datalake daemons
+ command: "{{ item.command }} chdir={{ datalake_deployment_dir }}/{{ item.dir }}/"
+ with_items:
+ - { command: ./bin/drms-daemon.sh start, dir: "DRMS-{{datalake_distribution_version}}" }
+ - { command: ./bin/drms-custos-synchronizer-daemon.sh start, dir: "DRMS-Custos-Synchronizer-{{datalake_distribution_version}}" }
+ - { command: ./bin/orch-api-server-daemon.sh start, dir: "Orchestrator-API-Service-{{datalake_distribution_version}}" }
+ ignore_errors: yes
+ become: yes
+ become_user: "{{ user }}"
diff --git a/ansible/roles/data_lake/templates/custos-data-synchronizer/config.yml.j2 b/ansible/roles/data_lake/templates/custos-data-synchronizer/config.yml.j2
new file mode 100644
index 0000000..62a2f6e
--- /dev/null
+++ b/ansible/roles/data_lake/templates/custos-data-synchronizer/config.yml.j2
@@ -0,0 +1,17 @@
+pollingInterval: 60
+dataResourceManagementService:
+ dbURI: "{{ datalake_drms_neo4j_uri }}"
+ dbUser: "{{ datalake_drms_neo4j_user }}"
+ dbPassword: "{{ datalake_drms_neo4j_password }}"
+custos:
+ host: "{{ datalake_drms_custos_host }}"
+ port: {{ datalake_drms_custos_port }}
+ custosId: "{{ datalake_drms_custos_client_id }}"
+ custosSec: "{{ datalake_drms_custos_client_secret }}"
+ custosBrokerURL: "{{ datalake_synch_broker_url }}"
+ consumerGroup: "{{ datalake_synch_broker_consumer_group }}"
+ maxPollRecordsConfig: 10
+ topics:
+ - "{{ datalake_synch_broker_topic }}"
+ tenantsToBeSynced:
+ - "{{ datalake_synch_tenant_id }}"
\ No newline at end of file
diff --git a/ansible/roles/data_lake/templates/data-orchestrator/application.properties.j2 b/ansible/roles/data_lake/templates/data-orchestrator/application.properties.j2
new file mode 100644
index 0000000..f2763e5
--- /dev/null
+++ b/ansible/roles/data_lake/templates/data-orchestrator/application.properties.j2
@@ -0,0 +1,21 @@
+spring.datasource.url={{ datalake_data_orch_datasource_url }}
+spring.datasource.username={{ datalake_data_orch_datasource_username }}
+spring.datasource.password={{ datalake_data_orch_datasource_password }}
+
+## Hibernate Properties
+# The SQL dialect makes Hibernate generate better SQL for the chosen database
+spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.MySQL5InnoDBDialect
+spring.jpa.properties.hibernate.enable_lazy_load_no_trans=true
+# Hibernate ddl auto (create, create-drop, validate, update)
+spring.jpa.hibernate.ddl-auto = update
+spring.datasource.driver-class-name= com.mysql.cj.jdbc.Driver
+spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl
+
+config.path=${AIRAVATA_HOME}/conf/config.yml
+server.port={{ datalake_data_orch_http_port }}
+
+drms.host={{ datalake_drms_host }}
+drms.port={{ datalake_drms_grpc_port }}
+
+mft.host={{ mft_api_service_host }}
+mft.port={{ mft_api_service_grpc_port }}
\ No newline at end of file
diff --git a/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2 b/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2
new file mode 100644
index 0000000..80b34f6
--- /dev/null
+++ b/ansible/roles/data_lake/templates/data-orchestrator/config.yml.j2
@@ -0,0 +1,17 @@
+eventProcessorWorkers: 10
+inMemoryStorageAdaptor: "org.apache.airavata.datalake.orchestrator.db.inmemory.DefaultInMemoryStore"
+messageFilter:
+ resourceType: "FILE"
+ eventType: "CREATE,MODIFY "
+ resourceNameExclusions: "swp$"
+outboundEventProcessor:
+ pollingInterval: 5
+ workflowEngineHost: "{{ datalake_workflow_engine_host }}"
+ workflowPort: {{ datalake_workflow_engine_port }}
+ drmsHost: "{{ datalake_drms_host }}"
+ drmsPort: {{ datalake_drms_grpc_port }}
+consumer:
+ brokerURL: "{{ datalake_data_orch_broker_url }}"
+ consumerGroup: "{{ datalake_data_orch_broker_consumer_group }}"
+ topic: "{{ datalake_data_orch_broker_topic }}"
+ maxPollRecordsConfig: 10
\ No newline at end of file
diff --git a/ansible/roles/mft/templates/agent/application.properties.j2 b/ansible/roles/data_lake/templates/drms/application.properties.j2
similarity index 58%
copy from ansible/roles/mft/templates/agent/application.properties.j2
copy to ansible/roles/data_lake/templates/drms/application.properties.j2
index b6a5260..72c7571 100644
--- a/ansible/roles/mft/templates/agent/application.properties.j2
+++ b/ansible/roles/data_lake/templates/drms/application.properties.j2
@@ -15,19 +15,12 @@
# limitations under the License.
#
-spring.main.web-application-type=NONE
-agent.id={{ mft_default_agent_id }}
-agent.secret=CHANGE_ME
-agent.host={{ mft_default_agent_host }}
-agent.user=CHANGE_ME
-agent.http.port={{ mft_default_agent_port }}
-agent.https.enabled=false
-agent.supported.protocols=SCP,LOCAL,FTP
-
-consul.host={{ mft_consul_host }}
-consul.port={{ mft_consul_port }}
-
-resource.service.host={{ mft_resource_service_host }}
-resource.service.port={{ mft_resource_service_grpc_port }}
-secret.service.host={{ mft_secret_service_host }}
-secret.service.port={{ mft_secret_service_grpc_port }}
\ No newline at end of file
+neo4j.server.uri={{ datalake_drms_neo4j_uri }}
+neo4j.server.user={{ datalake_drms_neo4j_user }}
+neo4j.server.password={{ datalake_drms_neo4j_password }}
+custos.id={{ datalake_drms_custos_client_id }}
+custos.secret={{ datalake_drms_custos_client_secret }}
+custos.host={{ datalake_drms_custos_host }}
+custos.port={{ datalake_drms_custos_port }}
+grpc.port={{ datalake_drms_grpc_port }}
+server.port={{ datalake_drms_http_port }}
\ No newline at end of file
diff --git a/ansible/roles/mft/tasks/main.yml b/ansible/roles/mft/tasks/main.yml
index 896c30d..f47ceb0 100644
--- a/ansible/roles/mft/tasks/main.yml
+++ b/ansible/roles/mft/tasks/main.yml
@@ -24,23 +24,23 @@
- "{{ mft_secret_service_grpc_port }}"
become: yes
-#- name: git checkout from MFT github repo {{ mft_repo }} branch {{ mft_git_branch }}
-# git: repo="{{ mft_repo }}"
-# dest="{{ mft_source_dir }}"
-# version="{{ mft_git_branch }}"
-# register: checkout
-# tags: update
-# become: yes
-# become_user: "{{ user }}"
+- name: git checkout from MFT github repo {{ mft_repo }} branch {{ mft_git_branch }}
+ git: repo="{{ mft_repo }}"
+ dest="{{ mft_source_dir }}"
+ version="{{ mft_git_branch }}"
+ register: checkout
+ tags: update
+ become: yes
+ become_user: "{{ user }}"
-#- name: Run MFT maven build
-# command: mvn clean install -Dmaven.test.skip=true chdir="{{ mft_source_dir }}/"
-# environment:
-# MAVEN_OPTS: "-Xmx2048m"
-# register: build
-# tags: update
-# become: yes
-# become_user: "{{ user }}"
+- name: Run MFT maven build
+ command: mvn clean install -Dmaven.test.skip=true chdir="{{ mft_source_dir }}/"
+ environment:
+ MAVEN_OPTS: "-Xmx2048m"
+ register: build
+ tags: update
+ become: yes
+ become_user: "{{ user }}"
- name: Stop daemons if running
command: "{{ item.command }} chdir={{ mft_deployment_dir }}/{{ item.dir }}/"
diff --git a/ansible/roles/mft/templates/agent/application.properties.j2 b/ansible/roles/mft/templates/agent/application.properties.j2
index b6a5260..01a19b4 100644
--- a/ansible/roles/mft/templates/agent/application.properties.j2
+++ b/ansible/roles/mft/templates/agent/application.properties.j2
@@ -19,6 +19,7 @@ spring.main.web-application-type=NONE
agent.id={{ mft_default_agent_id }}
agent.secret=CHANGE_ME
agent.host={{ mft_default_agent_host }}
+agent.advertised.host={{ mft_default_agent_advertised_host }}
agent.user=CHANGE_ME
agent.http.port={{ mft_default_agent_port }}
agent.https.enabled=false
diff --git a/ansible/roles/mft/templates/secret-service/secrets.json.j2 b/ansible/roles/mft/templates/secret-service/secrets.json.j2
index 3e9435f..cd34c6e 100644
--- a/ansible/roles/mft/templates/secret-service/secrets.json.j2
+++ b/ansible/roles/mft/templates/secret-service/secrets.json.j2
@@ -1,10 +1,11 @@
[
{
- "type": "SCP",
- "secretId": "local-ssh-cred",
- "privateKey": "",
- "publicKey": "",
- "passphrase": ""
+ "type": "SCP",
+ "secretId": "ssh_storage_preference",
+ "user": "ubuntu",
+ "privateKey": "{{ vault_mft_agent_default_ssh_private_key }}",
+ "publicKey": "{{ vault_mft_agent_default_ssh_public_key }}",
+ "passphrase": "{{ vault_mft_agent_default_ssh_passphrase }}"
},
{
"type": "S3",