You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/09/07 08:27:40 UTC
[airavata-data-lake] branch master updated: Generalizing ansible
vars, adding zookeepr role, kafka installation,
migrating to new backup instance
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new e76ee50 Generalizing ansible vars, adding zookeepr role, kafka installation, migrating to new backup instance
e76ee50 is described below
commit e76ee502d6c4f85629b856a2f33067c83b92b397
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Tue Sep 7 04:27:04 2021 -0400
Generalizing ansible vars, adding zookeepr role, kafka installation, migrating to new backup instance
---
ansible/inventories/test/group_vars/all/vars.yml | 15 +-
ansible/inventories/test/group_vars/all/vault.yml | 302 ++++++++++-----------
ansible/inventories/test/hosts | 2 +-
ansible/roles/data_lake/tasks/main.yml | 17 ++
ansible/roles/kafka/defaults/main.yml | 19 +-
ansible/roles/kafka/handlers/main.yml | 12 +
ansible/roles/kafka/tasks/main.yml | 30 ++
ansible/roles/kafka/templates/kafka.service.j2 | 14 +
ansible/roles/kafka/templates/server.properties.j2 | 195 +++++++++++++
ansible/roles/mft/tasks/main.yml | 14 +
.../secret-service/application.properties.j2 | 6 +-
.../defaults => zookeeper/handlers}/main.yml | 14 +-
ansible/roles/zookeeper/tasks/main.yml | 59 ++++
ansible/roles/zookeeper/templates/java.env.j2 | 1 +
ansible/roles/zookeeper/templates/zoo.cfg.j2 | 28 ++
.../roles/zookeeper/templates/zookeeper.service.j2 | 16 ++
.../{kafka/handlers => zookeeper/vars}/main.yml | 24 +-
.../airavata/datalake/orchestrator/OrchClient.java | 25 +-
18 files changed, 608 insertions(+), 185 deletions(-)
diff --git a/ansible/inventories/test/group_vars/all/vars.yml b/ansible/inventories/test/group_vars/all/vars.yml
index a397ceb..bc289a7 100644
--- a/ansible/inventories/test/group_vars/all/vars.yml
+++ b/ansible/inventories/test/group_vars/all/vars.yml
@@ -30,7 +30,10 @@ datalake_deployment_dir: "{{ user_home }}/data-lake-deployment"
datalake_source_dir: "{{ user_home }}/data-lake-source"
datalake_repo: "https://github.com/apache/airavata-data-lake.git"
datalake_git_branch: master
-kafka_url: 149.165.157.235:9092
+kafka_listener_port: 9092
+kafka_url: "{{ groups['data-lake'][0] }}:9092"
+zookeeper_client_port: 2181
+zookeeper_connection_url: localhost:2181
mft_deployment_dir: "{{ user_home }}/mft-deployment"
mft_source_dir: "{{ user_home }}/mft-source"
@@ -43,7 +46,7 @@ custos_repo: "https://github.com/apache/airavata-custos.git"
custos_git_branch: develop
mft_default_agent_id: agent0
-mft_default_agent_host: 10.1.0.42
+mft_default_agent_host: "{{ ansible_default_ipv4.address }}"
mft_default_agent_advertised_url: https://beta.iubemcenter.scigap.org:8443/downloads
mft_default_agent_port: 3333
@@ -62,7 +65,9 @@ mft_resource_service_custos_client_secret: "{{ vault_mft_resource_service_custos
mft_secret_service_host: localhost
mft_secret_service_http_port: 8081
mft_secret_service_grpc_port: 7003
-mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.file.FileBasedSecretBackend
+mft_secret_service_backend: org.apache.airavata.mft.secret.server.backend.custos.CustosSecretBackend
+mft_secret_service_custos_client_id: "{{ vault_mft_resource_service_custos_client_id }}"
+mft_secret_service_custos_client_secret: "{{ vault_mft_resource_service_custos_client_secret }}"
mft_consul_host: localhost
mft_consul_port: 8500
@@ -103,7 +108,7 @@ datalake_data_orch_user_group: emc-users_e8a37f6d-e3b3-4e4a-9081-265bb42b1b99
datalake_data_orch_admin_group: admin-group_c1aac070-1512-4e98-b1d1-e06a94f03665
workflow_deployment_dir: "{{ user_home }}/workflow-deployment"
-workflow_zookeeper_url: localhost:2181
+workflow_zookeeper_url: "{{ zookeeper_connection_url }}"
workflow_helix_cluster_name: datalake
workflow_datasync_wm_name: datasync_wf
@@ -117,7 +122,7 @@ workflow_worker_kafka_consumner_group: mft-even-group
workflow_manager_host: localhost
workflow_manager_grpc_port: 6565
-workflow_manager_mft_callback_url: 149.165.157.235:33335
+workflow_manager_mft_callback_url: "{{ groups['data-lake'][0] }}:33335"
workflow_manager_mft_client_id: "{{ vault_workflow_manager_mft_client_id }}"
workflow_manager_mft_client_secret: "{{ vault_workflow_manager_mft_client_secret }}"
diff --git a/ansible/inventories/test/group_vars/all/vault.yml b/ansible/inventories/test/group_vars/all/vault.yml
index 3f44c69..38bf6f8 100644
--- a/ansible/inventories/test/group_vars/all/vault.yml
+++ b/ansible/inventories/test/group_vars/all/vault.yml
@@ -1,152 +1,152 @@
$ANSIBLE_VAULT;1.1;AES256
-30326232633238616564666334386437643339373166663630326631653866376134376435373134
-3936643533306432383836383462643233313933303463630a613962663466626461633465323963
-35303936303533313564636539363933353166373031356334656535653935633633336364613364
-3364383532316163300a623736396564636237306431303630343334353563333732323364343761
-30626439353035343236343262663538633262343162343234646434666334643631626662353261
-30363362633533343562656231366132396637373135306132336139306331323734366261663134
-39323564663332356662626562306263383133616663663131366433396431356663343962636661
-64643363313261346433653230346161303330326130646662363838643230366636343035383336
-31383239323066376132303537333834663632323336353130626636343138356435323836333235
-63343139313562636436316532376135373636333237366333306534303836626339363930666662
-61303237333130326631373032333632623831393130616131646136393932626561356230393565
-34626234643839653361613637663063633661396163363030396333333864663463636632373830
-36646261643535623466343031346231663833373462663531313166383262663330616161333033
-64303634386662656533393064613964356163323632356562663666386336623532616138343339
-64633932383430303962613362386235343930643432663030653730363264333637666536666164
-38343032323135343063323565343963636437663135383834383164386238353864656330303637
-63623765366362303335393338363033383739306132303731393938663138343161336431353535
-37663238666166386165366639623262356466323633643762316465353536373337376131313466
-31653938386363636634383237613065323366336532383431373331346232363766616562646361
-30336330643437336432383563393936333233366332376436656466323464366566346266346134
-62636563623764356230623664393935363161633430333362356436373666336637353438666532
-63396632666466373034663034373736663238623032326132313832303363343764303835623934
-33303735346130646237366565656138373034613739396663306133346366383262333134353434
-30333639383530336138646135326561306234353761333732623632313431393037373132626363
-66363132373965333632363333336535376462623933366331656464343032393536363234383162
-62666437393437306633623833386664373633623562653533376535646534336331666134633866
-37353936373338363238303239333538393561646265646135376661353064323565353834303936
-66333362653730353564643534373930626331313937323862623130653532646534326362373936
-62356338386362653132383836326265626230396632393333663464373463633331376566663331
-32663834313034346434336439323764633533366366643837376565353361633933613337326163
-63656262613461383539636631646561613637386630623634336133666139316631343433303034
-63396237383964633938643461376164336162333931643964386361393765356231633038653230
-35643766366334643365383366663031323235613730623232326439623739306535303139353735
-66393830656666353231636538366661636339326535316332626533366530323939653563366238
-33646432343536336662323738353464353232313764353064633534633964326166376234626465
-36656330303430663536373030343433373239616164616534643163353865613334646361323439
-39303865643166666439623133356139636433373564333832616663663339646462323031643039
-36346438623235666263346366623630623239666131326336346336653735633139623964326338
-36636239623731333166353865356434666538376536333661336161366463393334383266343732
-33323064303733646332303535336562633064336337383839323430373966613634366635663031
-65613737366263636433616239376665636239373331653138623634613032616137626630646230
-35343863346235343164393730666461353031363832343233396634653038613638663933363238
-66623835653163376564353963363531316462343536346366343539316464383932393266656238
-38616131616430343137663264653030303737373739363861633732353932613631313539343837
-66666434313463383666633961313966393762396632323237363632663163353133333237396461
-62613566356437396537316261656338313839386537313338333636306435623334623531626663
-39666139363034396333396264633830333963663265343439383364646236363137303434343830
-31353837383332313864656565663465383433313338616135333738303764323964626533653261
-37623362636536333136306435653034343531633531653537666537363465373632666164343838
-37333435396266383762313439653839653236323939313462636462383264643832353330396464
-37653133346663643130326666396633333331623838336331626631663435363232653662663431
-65363663363237653462353238316633353038613130646465656666653263313835353332323331
-35306232396632663933393837663030316666623333616438613263643336393632366665363033
-65636261363433666666663462303936393937333762316531386438613162353064663661653662
-37666636643861396536396337626334616437393733333961373561396562306235623735613763
-38346436373834633037376166653032653861316231323537623062343661326136663735646332
-36346438323136353163333934343764666638383938336162623932393934396439396238373632
-61356166643835663139616531386439343633623934653664623730393732336139303934633135
-36663763643433633036373934623965666263346465316335643364316436623763383666643238
-33303165393635623936393561353132333132626238626165623761303435333838656666643566
-64326134383534626334323566336530303338613639363139646465646365333734666561393639
-31326566373531386263346533333564393632393238303135363662666264366164383835653437
-37363431303737343863343534653137613161663165393936393035643730303830363931393563
-64383233313834633762616439653338663063376261623662363632346336363361343631343966
-34643039633062393938653438353861356365656464656165346131323064353362643837666434
-31663332626637653732663764633538326565373734636236313331386561363165643938643636
-36363332613361303364653533326562323161313161376431643163366532306530643263373431
-65303162666164656639653633613338636566666163393639613065383131616563616631363366
-30393834323663336465343562623166323462323834663736353639613930646461306530636161
-34613832303930666638363261646263313434633539356532323166623165356665636431323133
-62636633613763303565333637363163333935636363626333316536343034323331343766393935
-39303635373938326161323135663231383464326564343734616133623137306532366139393031
-64393563316364386666316466313866316261663262663639643630663036356537636530633434
-35373564633432333864376161386532643366346339653438366261396631306362623533346664
-34363465323761356336643062343734626562383764366131653330353834656564303663393765
-61656437383963623732326539383361333364356435376231366464623661376132336361303138
-65653264633065346264393632613437663238303965363266383264306638613134373730346364
-30643436663531613139643333626262376237356632333662393032396265393135353834373833
-66663564643861313535643037346230363462653337373366316639633563313064643530613630
-33323461373961623661353137643637653239383262643363316432643364303965376231363939
-38366264326466313837623732393634316263316464326637383764333935326235386531653131
-35376264333231643330633765656537363730386162643131383531346432373933356261386236
-31353532363763626431643064633136353834373163646133636663363130343764313334646163
-37646261326537646439363032316564333637643036373637303535356663633834343532343736
-33643531393437323835616330383833613636383533333232373566366536346662623730396266
-63313437646336363462623363633635323634396337313931636338383736383636396539653438
-35386532313930343161313632646635353461386631653732626233366263353136306636323762
-30623664383534623462353263343230623065333062643233666533313665663662373936633430
-31343738393933336262636438333139343731376561313265316138353739313231346165646230
-39656236303736363436353434316130346539343930353836616262323731353961376562623030
-37376261623665313565663130643565393733326661656466353334643761633635633962353062
-61663437613161343030646565306361313363303030623235316438343333333163656236653038
-37613164393234396239343462663664356665313238336137656334623933386638316561393531
-63393563636237376261363838303634333931373334363338313436643730613738626334333630
-62393731663937313063643434626632343066336230383336383561633833376666373337353738
-61323533386337303132623037643138653966376238666132646339326564383964643533333061
-63313264336433633035323438306638643238623661356230313262613166643462353935646632
-31656161626663393365656565666666643739613661633135616239373136656463393639303064
-35613465303238383132343764356434356130353131303230656536366539616665316537646235
-39373965303938353935356163303963666165356532303165616435666463613362613633653933
-35616636633835393463376161303033636332333735666430393665363635373264303630323430
-36633761383464623331346434363336333534313033653331633731613136656337643461346362
-32656661376164643761626666373837633037663735326664626664616636363161646633393737
-65623731383539396530383532383134343830616435393030636364663634373539663063643830
-34633432363463393361386134366234323636626161333165313836346463373532633162306130
-63386437306566353232373631623033316437613866353335303364343364633463623962326239
-30666566656235373833323261613265663036626434373033633832336139393765613732363731
-61393736346235666236623437373162613532383934646130643039383337653737373030336163
-36653132393737613862333138353931643231386230366636613464613866666132623663346266
-62373033363435346332336564636232383732333635633033353164363631313663323434376533
-35353130386335643639663162386230323230646161346632656163313439383437313230343362
-37326365316436393632383330396337316139376439316661616535303739363761363730653431
-66316163316462646539383663336439663065643466626134646132383766623731313830653933
-30313039343438626566626630653538643033653938313937383437663333306331313631666131
-32386338323438363237656232386435643861313664353734366565653935333430393563353432
-62393562373731653361663537363838633165393937626136326466393838393764656463306461
-33386332306235643534663231326533373038383037643739326662313365326665313137323830
-32366433613638373066306438663365323263366465383530356565633933366465303762386264
-61313765326437313832666565306534626339373539363132336662636563343037396135376434
-34396661646261353335643936643665643134633732656665303137663537663433643465616666
-37656539653834643331633835396430353263636336336136316261306464333064613631313132
-63303238396134626435363232356533386166396563373163363436653661646632393863363435
-33373233333138373830653939653338396433306234383331633733643365376565313466303839
-39663531333034663466336664373463656665393132616332303032643766633738356431323435
-34313530643232366536323465353564626132383262306637636534633434353633656164323662
-61633061646632666135366661366631613165636131373863626165373062666665653237656362
-61613062363134393730353538323361616466356435326366333363313536383062303432303164
-36343138373866633766356364393030303832653061326536326133306433326136636437616536
-31366138373165306333623563323863323039323137653030643634386165653861313835663335
-62643639373662366331353662623639343339373834323564636637326661316561663831636163
-32666364303666303466653462616138623938646336643736336665613162313666376538383936
-62653332313530313064306638343761653964393231323863633831303737646433616630623862
-65353861376462383431656266313239373837613132633033306263653134383865333039346332
-61613734316137346563656365363433303266383936613634646164316562396464653536316537
-34633238383335333535376134346665646462653132306238366336643139656538323033376431
-63643631623331373934633963666165616365656565363166373865323933656264303664636333
-35383838373935306435333466373530623837623966656130666339353361656238323861333531
-35653662316163356533343231616364656562336632376566333638333563383666363134613363
-31623034346664393933383830336636353337653663636433306636303164323532656235643235
-33383938613338333733303231356165643832656264663332313365623636653130353466363766
-36623337623338633735356235636264346366376462323635386165343535343239343633353937
-61343665383530303332303034303234653832633232373038336336653165303639633933623738
-35346465353662643833653265353733346664663738396336363833633464393230613139643031
-31373231626233613235303431616263396631363535656638633264393961323235613433306566
-31353334323239376534393937613038616337623539336636663066353939373930633830336364
-62343732383435626466643464376236656339313635326137383766363964636432316635383666
-34663865313063323566356330333462623665326232333261393233343861643036353934323063
-63383834626636633439393335323766643433353566663163373830623836626263623332356163
-35343932323033336338343863343633313562643135663038353438653634636464613736336230
-64643139653432383865346539306433373236373036613238383232373362626437323566333737
-613239626335393738353861313831313935
+66616333386133663333336635326631653766333039323964333238343863616465306662386537
+3338616131306236306430636465396566323234656336300a376436376335666632376662323466
+33366163366438623630336662633132336139316437313561383462363332666232663833626435
+6538616162613537610a623036343162666565646630333931393663386462646433306535626462
+37663738323465663235396332613536633737393238326235663561663435343164623332366437
+61653135613730346339343466653830326565663862353338616236376139616265303366623763
+36323836653762376339366533376562643030356662393139336537376363663235366334313663
+66336633663161396335356162653066376664336435383730643531383365623038373637653536
+32376133333438323466386261353436306433636631346238383466616638623137346532333264
+34653339386263636336383536373630316531646434623063623538666636633832396537343063
+33356638336337346430613663323536666665396461616639363533666264613733333737333031
+66323436333962646666393932643432386264333564323137303336613935626563346466386564
+63323034336262633838303935306134656362336535353061323836356537346464323136353132
+64646335343738633239636539376538653739393165333265653863383839643161316536393536
+61303237306136613334643334663331333535373765353636356466646439613766656433343938
+33343136343932323838633034336533623839313735383739356136653666643437393634366366
+39336535323339666263346239313931316133633930616133646662386630303734313630396162
+65343737646139376362343061363036333434656539366462376232636433396361373934663466
+39626664366137626434666238383862373466353533393664353230646432663533643936666130
+62343964373466626230646336333166303135333331343661336362393566653765623762613938
+32356134323332393464353131343763383861613238363634633136333737303133646161656635
+31353037343736383238343937383331646531623163613339616666653964323335623034303836
+63386362323065393266643464653132623935626562316434316466323238323061613939316436
+37643530376363633539653033326136313735646466613539393763386537313632393036623135
+61346231626230353233366130643636343463623465353365646639353237653063373537663766
+31363163393334373432643462393233666438323334303232326466383337343538656662613338
+39636633633838316537393433633263343433643039336264653563643064313339303763373962
+35393966633136346362383636633736653963303262623331666664666235303234306365656437
+33383236616165646236313063323562383039396531643632323937336533393663643039613139
+66636637356164363136646132663734353862623737343033303134316431363834313064653933
+34343738343431366139623665343437346364336139373039363634316437623331623233326164
+35323737383362636166643365393636393437393435626133376562323635646662356463343265
+65623737343737303639633331326639323461343336316332363932306536316434363464333134
+30386431613531383365323733316266303162316134623034396530363864336639656338383366
+39333162383764643131396434306330616537366536396262353865306666643961383232633539
+62653833316139656236373134323431623537326237636633666661373039656133356630383962
+32633965376365316235383631303965386163313631643665626434643033646433626661383137
+65363432373338643466376238306336656264383937313061323135376637336439646466303065
+32356339623365613166336363363234353330383763316235353465383631366161626466303530
+35636362323537366439653364333163366464386565376537613330366431643765376339366233
+33396539373564613036363961363064343030353237366636336537383432303162373665653936
+61366134626564376536373238646664333234353466303334346266346461653434653838323965
+65336561613961356464393133383032393163626637393633386335666633636134336362316538
+33313163326138653836346266393465613233313935653266363837303837623432313938346432
+61633132363538356266353132343037653666613462653033303530643565346238346633663764
+37656339643637333536663232386233663634333837393433316564386635383430336163313761
+33393464613535306563306434353038356562336532316637666634653664346432353233633332
+66336233373363363765626234633835393531353831643163333064636461356337613038623663
+35336230653338623436306530633030363736373763636635623265323234316234616639363562
+30316138396130613339343934653039376361333234356661386237303861323936643663653533
+31336138653132653732663032646135373532323239666230616438386363316431363031376163
+36316262336437626434623637393466303635356639383839393935316463383539383333323238
+34616264323563343361303662666139373662336338313934663633336431633137323663376532
+35626463636266636336633863646630633731666166393031636234666562306336643762313761
+33313866303634316465336236343833373663366334396637663361666461326630313830303462
+37623863636430633230646261303262343135646333373530333938353237656463393661646161
+32306238303566356634306561616339656237373331613563616365653232373634623435343266
+30363035666265346139363639343465333064626231323665383865653266666461303332353662
+32316138343964393165373439323632333330393231303633653666653033393137366364303661
+34353939636266643637313666343136356437353530373430623138313566633533383630343936
+35386232366337383236646437626134366439646663386565613635363766396565653332623761
+36356331623938643237376337653763396436643038393238333639333836383561356131633230
+37356363633430333231633465373963386634373932613166353636623430643834383732386436
+62356233326336313636616561653838616230383834383237663830663639393263303536333763
+33633438663139323835376462303066393139663964656265653837383332663133326635326336
+37383962626666396333346135653835653738343462306130323961323463383133333030636131
+32663463393538666534626637353939356164383861636236386366643136343536383737343964
+63363837363164396264353538383232633539326637353662366666323663653332656239346233
+33376436363062363439636563663966393536613834663561323762656464366434346264666130
+37363263323165316135626266626234636236616330663930663732646263656136656235306163
+62663964376465613863396631633234656132396338623365663161313434366465346331396461
+63313132323765386134316131666334303761626664356633363763653631633339333535653839
+37643339323665303066613138383231303066663435323237656231313039363361653538396662
+32356230313866333530653733346132346666383231393232626532303132306434656537323537
+32363333613732366432303263333866393061303366646638323735663263616531656632343662
+39643262613664636461653563346438373438383530666664623336376637353131646633373838
+35636561643632626663323331363362363630366166376133396366313034386235396539386438
+36633936356639643265666133353661306334336631653938316331656662343831306661346538
+66366333353538656634363330633738623537656531633938376338323537366137646438633664
+35316530363436316538383635343231323230303762386631363863633336326466353163616439
+62623063323230313136306536303935323635623133366266656239306466313438373733356331
+66393134323834396662643438336431613461396665616331376430613033306639636364613537
+36346430646662646238366462366463653433333361323461316265366333613533633132353734
+62636462396531656333623966393534636164323334366638333636636533623264383730663032
+33626638653535323434306265383634333734616434616439376631386231653739303735366530
+36363261653065336230623162653537363364663965333331656539333062343066386637333161
+36646535393932646539393139626634346430623736386437343636343361383739356539633736
+61613432306233313438373161376434613737306337653634383732353833303561666132323563
+64353161396665346463306464666136363065656137386366616561616362323530623864303161
+66663865623565643263633864316134646133633436643934306637386430666433613065346666
+37376239303239313966366135316236393030623232633933636639386337366231633266303263
+61393166376464333739616637653733646337363661383961656262393232653030383963623231
+30363231373764393833373164306531656432626533656637363962316434316530303338306236
+36323863343530656361306231643336626533613530653338663734653539643732653262346239
+61663831383736386539636265653236316639663963623065393630343538316366636432663230
+37306366313832666538386234386561656563306135383866623866626535613135663131363535
+33313537646536383361336661396232633931326636356465383366396339646565323865623663
+31336364373930653663666231313164346434643732633130653437343530646262363434386263
+33643832333065383362613437356630663362343537376461313731346333366231373831396364
+64306332623866653062306562656666346231306438383761323138323065363234316364346164
+61616465303637383566316662646437393837643263633930613133366165393862633934346164
+34613661613663653961316239303766366636623438666533643834393839376566333138326261
+38393961653935386131373131393831383932343233366334383831653162393762386261396330
+64386339666539386132643935626365643536613234336438336661666532386466386566363934
+64353361363461346633376462346462313134626437346338663034303037326666623066626231
+32356663313036623662346663663236386536336636373839363561633839613837356630653631
+61366139306361316666623663353062326231353632356565633933383438643130376638616632
+35333239333938376637376236623965376163353437336234613539626137363536656639623835
+34393766316637316337636631363235383931643638383166366465623862383161636334313033
+66313737383065663532326538333732313166353232363366353262303839343361363536346330
+30643739663434386333653731353230616537353865396338303138356239336533623330666334
+30356365636566373030626538646333626432663334323530613634663464303366383431383238
+39313434323336646332353235396262636638313538663634666136663539396161356461353265
+30363534303462323734623962373738396437313433366464646132636563656161393966336563
+64643233613363656335636333646432373635386330373561323664646337393635363861313366
+39653232303533663165323465303866656362663335313666393835653766396533303666333338
+37373531343765616661363064653362363061646231376236373237646130306566313432333039
+36303764316533653066383630356136663638636139366437636532323934383937303537366536
+31313862326335323566313837656462633830653061323966336330386166363762346162386637
+39393933383033393930343737646135616436346366333839653735316333633036366238356638
+35353732363538386362643137316530333166343032653132393234336365363037656437363138
+62303631366461373835623166316237636136633265333464643738633466363031393430613239
+63356439326634373130623031393539663936306566306263313263663335356433303332316138
+61353966356163313537636464393662663436376564346637306536333130633762373534646632
+63393362396462306631646331383634343837393639356230343630373063356535306265613566
+39373839363139653563666131303638393734353863353833396632346133663361646338363133
+36383836663136616266383965336531666431663963326364623634316364663438663531373366
+38363566353961366634646439366264323565643566636432363431383230353537663738323936
+62343438653661643931303338666662613361333130626531613166366562363932303763373866
+33386132646163316534393366306633353661316630613234323532663037636365616438643332
+64323062363964313430363236353532316337323034326239623566306233323636383563623135
+63383765663164306663323165346235393032616330353865653132393466316130373637303034
+65386233373463306265306661393962363866656265343037663062323562303930636561326465
+32313132303430656663616264313337326138636662666234623533653839353130656539386435
+31656330326633646331646463343864353061323234623237616136383734326635303264316134
+64326331343262613262666637353036313636643764613232376538663834366230326539623731
+37623039353933633230613037346339633339633263656239376139303436663565333930336263
+62353739633030353263636633613738616432326661363039316134656563623636303366353637
+36636533643663633763326531313233323837363166633933376265613735306439316232373335
+62383534323235383839623339363165333432343030393563363163613335653435363238613363
+37653866316663316435653239306439633364633633386138376536386231343465306563393031
+30313339346561643531303864653465383730373933613434343634366164643762373039663436
+37353665313431336562303830346532313966663163663831646262326236626230313566393438
+65376162663861663064386336313238386131623264363737666630373437306630636434363263
+31393833396434623830333564656466313933316664653432646130396663343935366633396138
+65643662303531356138633663623534636237356163343962653032356461383438306632363331
+38643235386262393063623562666563623761353136303538376533393432633665636665613861
+64333865336530643562326530626634356563376636396236323734396231346666626566343062
+39366665393230656165633865323462356233313432336338623935363733333761313135346166
+63636365666566356462653030663838343837616365653766626632313136393631616638366332
+353738306663323333383861383064643863
diff --git a/ansible/inventories/test/hosts b/ansible/inventories/test/hosts
index 90255bb..337db52 100644
--- a/ansible/inventories/test/hosts
+++ b/ansible/inventories/test/hosts
@@ -1,2 +1,2 @@
[data-lake]
-149.165.157.235 ansible_user=ubuntu
\ No newline at end of file
+149.165.156.22 ansible_user=ubuntu
\ No newline at end of file
diff --git a/ansible/roles/data_lake/tasks/main.yml b/ansible/roles/data_lake/tasks/main.yml
index 1e4c224..c8771b1 100644
--- a/ansible/roles/data_lake/tasks/main.yml
+++ b/ansible/roles/data_lake/tasks/main.yml
@@ -33,6 +33,11 @@
zone=public permanent=true state=enabled immediate=yes
become: yes
+- name: open firewall port {{ datalake_data_orch_grpc_port }} for Data Orchestrator gRPC connections
+ firewalld: port="{{ datalake_data_orch_grpc_port }}/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
- name: Create Datalake deployment directory {{ datalake_deployment_dir }}
become: yes
file: path={{ datalake_deployment_dir }}
@@ -77,6 +82,18 @@
become: yes
become_user: "{{ user }}"
+- name: Delete lib directories of the distributions
+ file:
+ state: absent
+ path: "{{ datalake_deployment_dir }}/{{ item }}"
+ with_items:
+ - "DRMS-{{datalake_distribution_version}}/lib"
+ - "DRMS-Custos-Synchronizer-{{datalake_distribution_version}}/lib"
+ - "Orchestrator-API-Service-{{datalake_distribution_version}}/lib"
+ become: yes
+ become_user: "{{ user }}"
+ ignore_errors: yes
+
- name: Copy Datalake distributions to Datalake deployment directory
unarchive: "src={{ datalake_source_dir }}/{{ item }}
dest={{ datalake_deployment_dir }}/ copy=no"
diff --git a/ansible/roles/kafka/defaults/main.yml b/ansible/roles/kafka/defaults/main.yml
index 2a095ce..8793de2 100644
--- a/ansible/roles/kafka/defaults/main.yml
+++ b/ansible/roles/kafka/defaults/main.yml
@@ -23,4 +23,21 @@
kafka_package_name: "confluent-5.3.1"
kafka_tgz_url: "http://packages.confluent.io/archive/5.3/confluent-5.3.1-2.12.zip"
kafka_dir: "{{ datalake_deployment_dir }}/{{ kafka_package_name }}"
-kafka_rest_proxy_listener_port: 8082
\ No newline at end of file
+kafka_rest_proxy_listener_port: 8082
+
+broker_id: "0"
+network_threads_count: "3"
+io_threads_count: "8"
+socket_send_buf_bytes: "102400"
+socket_receive_buf_bytes: "102400"
+socket_request_max_bytes: "104857600"
+num_partitions: "1"
+num_recovery_threads: "1"
+offsets_topic_replication_fac: "1"
+trans_state_log_replication_fac: "1"
+trans_state_log: "1"
+log_retention_hrs: "168"
+log_segment_bytes: "1073741824"
+log_retention_check_interval: "300000"
+grp_initial_rebalance_delay: "0"
+kafka_listener_port: 9092
diff --git a/ansible/roles/kafka/handlers/main.yml b/ansible/roles/kafka/handlers/main.yml
index 761e265..9856989 100644
--- a/ansible/roles/kafka/handlers/main.yml
+++ b/ansible/roles/kafka/handlers/main.yml
@@ -20,6 +20,18 @@
---
# Kafka related handlers
+- name: start kafka
+ service: name=kafka state=started enabled=yes
+ become: yes
+
+- name: stop kafka
+ service: name=kafka state=stopped enabled=yes
+ become: yes
+
+- name: restart kafka
+ service: name=kafka state=restarted enabled=yes
+ become: yes
+
- name: start kafka-rest-proxy
service: name=kafka-rest-proxy state=started enabled=yes
become: yes
diff --git a/ansible/roles/kafka/tasks/main.yml b/ansible/roles/kafka/tasks/main.yml
index 40360d6..f152770 100644
--- a/ansible/roles/kafka/tasks/main.yml
+++ b/ansible/roles/kafka/tasks/main.yml
@@ -49,6 +49,17 @@
file: path="{{ kafka_dir }}/logs" state=directory owner={{ user }} group={{ group }}
become: yes
+
+# Config kafka server and start
+- name: Copy kafka server properties file
+ template: src=server.properties.j2
+ dest="{{ kafka_dir }}/etc/kafka/server.properties"
+ owner={{ user }}
+ group={{ group }}
+ mode="u=rw,g=r,o=r"
+ notify: restart kafka
+ become: yes
+
# Config rest proxy and start
- name: Copy kafka rest proxy properties file
template: src=kafka-rest.properties.j2
@@ -58,11 +69,25 @@
mode="u=rw,g=r,o=r"
become: yes
+- name: open kafka proxy port
+ firewalld: port="{{ kafka_listener_port }}/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
- name: open kafka rest proxy port
firewalld: port="{{ kafka_rest_proxy_listener_port }}/tcp"
zone=public permanent=true state=enabled immediate=yes
become: yes
+- name: systemd install kafka service script
+ template: src=kafka.service.j2
+ dest=/usr/lib/systemd/system/kafka.service
+ owner={{ user }}
+ group={{ group }}
+ mode="u=rw,g=r,o=r"
+ notify: start kafka
+ become: yes
+
- name: systemd install kafka rest proxy service script
template: src=kafka-rest-proxy.service.j2
dest=/usr/lib/systemd/system/kafka-rest-proxy.service
@@ -73,6 +98,11 @@
- name: Reload systemd daemons
command: systemctl daemon-reload
+ notify: restart kafka
+ become: yes
+
+- name: Reload systemd daemons
+ command: systemctl daemon-reload
notify: restart kafka-rest-proxy
become: yes
diff --git a/ansible/roles/kafka/templates/kafka.service.j2 b/ansible/roles/kafka/templates/kafka.service.j2
new file mode 100644
index 0000000..afc0118
--- /dev/null
+++ b/ansible/roles/kafka/templates/kafka.service.j2
@@ -0,0 +1,14 @@
+# {{ansible_managed}}
+
+[Unit]
+Description=Kafka
+Before=
+After=network.target
+
+[Service]
+LOG_DIR={{ kafka_dir }}/logs
+ExecStart={{ kafka_dir }}/bin/kafka-server-start {{ kafka_dir }}/etc/kafka/server.properties
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/ansible/roles/kafka/templates/server.properties.j2 b/ansible/roles/kafka/templates/server.properties.j2
new file mode 100644
index 0000000..ae27bb4
--- /dev/null
+++ b/ansible/roles/kafka/templates/server.properties.j2
@@ -0,0 +1,195 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# see kafka.server.KafkaConfig for additional details and defaults
+
+############################# Server Basics #############################
+
+# The id of the broker. This must be set to a unique integer for each broker.
+broker.id={{ broker_id }}
+
+############################# Socket Server Settings #############################
+
+# The address the socket server listens on. It will get the value returned from
+# java.net.InetAddress.getCanonicalHostName() if not configured.
+# FORMAT:
+# listeners = listener_name://host_name:port
+# EXAMPLE:
+# listeners = PLAINTEXT://your.host.name:9092
+listeners=PLAINTEXT://{{ ansible_default_ipv4.address }}:{{ kafka_listener_port }}
+
+# Hostname and port the broker will advertise to producers and consumers. If not set,
+# it uses the value for "listeners" if configured. Otherwise, it will use the value
+# returned from java.net.InetAddress.getCanonicalHostName().
+#advertised.listeners=PLAINTEXT://your.host.name:9092
+
+# Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details
+#listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL
+
+# The number of threads that the server uses for receiving requests from the network and sending responses to the network
+num.network.threads={{ network_threads_count }}
+
+# The number of threads that the server uses for processing requests, which may include disk I/O
+num.io.threads={{ io_threads_count }}
+
+# The send buffer (SO_SNDBUF) used by the socket server
+socket.send.buffer.bytes={{ socket_send_buf_bytes }}
+
+# The receive buffer (SO_RCVBUF) used by the socket server
+socket.receive.buffer.bytes={{ socket_receive_buf_bytes }}
+
+# The maximum size of a request that the socket server will accept (protection against OOM)
+socket.request.max.bytes={{ socket_request_max_bytes }}
+
+
+############################# Log Basics #############################
+
+# A comma separated list of directories under which to store log files
+log.dirs={{ kafka_dir }}/logs
+
+# The default number of log partitions per topic. More partitions allow greater
+# parallelism for consumption, but this will also result in more files across
+# the brokers.
+num.partitions={{ num_partitions }}
+
+# The number of threads per data directory to be used for log recovery at startup and flushing at shutdown.
+# This value is recommended to be increased for installations with data dirs located in RAID array.
+num.recovery.threads.per.data.dir={{ num_recovery_threads }}
+
+############################# Internal Topic Settings #############################
+# The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state"
+# For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3.
+offsets.topic.replication.factor={{ offsets_topic_replication_fac }}
+transaction.state.log.replication.factor={{ trans_state_log_replication_fac }}
+transaction.state.log.min.isr={{ trans_state_log }}
+
+############################# Log Flush Policy #############################
+
+# Messages are immediately written to the filesystem but by default we only fsync() to sync
+# the OS cache lazily. The following configurations control the flush of data to disk.
+# There are a few important trade-offs here:
+# 1. Durability: Unflushed data may be lost if you are not using replication.
+# 2. Latency: Very large flush intervals may lead to latency spikes when the flush does occur as there will be a lot of data to flush.
+# 3. Throughput: The flush is generally the most expensive operation, and a small flush interval may lead to excessive seeks.
+# The settings below allow one to configure the flush policy to flush data after a period of time or
+# every N messages (or both). This can be done globally and overridden on a per-topic basis.
+
+# The number of messages to accept before forcing a flush of data to disk
+#log.flush.interval.messages=10000
+
+# The maximum amount of time a message can sit in a log before we force a flush
+#log.flush.interval.ms=1000
+
+############################# Log Retention Policy #############################
+
+# The following configurations control the disposal of log segments. The policy can
+# be set to delete segments after a period of time, or after a given size has accumulated.
+# A segment will be deleted whenever *either* of these criteria are met. Deletion always happens
+# from the end of the log.
+
+# The minimum age of a log file to be eligible for deletion due to age
+log.retention.hours={{ log_retention_hrs }}
+
+# A size-based retention policy for logs. Segments are pruned from the log unless the remaining
+# segments drop below log.retention.bytes. Functions independently of log.retention.hours.
+#log.retention.bytes=1073741824
+
+# The maximum size of a log segment file. When this size is reached a new log segment will be created.
+log.segment.bytes={{ log_segment_bytes }}
+
+# The interval at which log segments are checked to see if they can be deleted according
+# to the retention policies
+log.retention.check.interval.ms={{ log_retention_check_interval }}
+
+############################# Zookeeper #############################
+
+# Zookeeper connection string (see zookeeper docs for details).
+# This is a comma separated host:port pairs, each corresponding to a zk
+# server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002".
+# You can also append an optional chroot string to the urls to specify the
+# root directory for all kafka znodes.
+zookeeper.connect={{ zookeeper_connection_url }}
+
+# Timeout in ms for connecting to zookeeper
+zookeeper.connection.timeout.ms=6000
+
+##################### Confluent Metrics Reporter #######################
+# Confluent Control Center and Confluent Auto Data Balancer integration
+#
+# Uncomment the following lines to publish monitoring data for
+# Confluent Control Center and Confluent Auto Data Balancer
+# If you are using a dedicated metrics cluster, also adjust the settings
+# to point to your metrics kakfa cluster.
+#metric.reporters=io.confluent.metrics.reporter.ConfluentMetricsReporter
+#confluent.metrics.reporter.bootstrap.servers=localhost:9092
+#
+# Uncomment the following line if the metrics cluster has a single broker
+#confluent.metrics.reporter.topic.replicas=1
+
+##################### Confluent Proactive Support ######################
+# If set to true, and confluent-support-metrics package is installed
+# then the feature to collect and report support metrics
+# ("Metrics") is enabled. If set to false, the feature is disabled.
+#
+confluent.support.metrics.enable=true
+
+
+# The customer ID under which support metrics will be collected and
+# reported.
+#
+# When the customer ID is set to "anonymous" (the default), then only a
+# reduced set of metrics is being collected and reported.
+#
+# Confluent customers
+# -------------------
+# If you are a Confluent customer, then you should replace the default
+# value with your actual Confluent customer ID. Doing so will ensure
+# that additional support metrics will be collected and reported.
+#
+confluent.support.customer.id=anonymous
+
+############################# Group Coordinator Settings #############################
+
+# The following configuration specifies the time, in milliseconds, that the GroupCoordinator will delay the initial consumer rebalance.
+# The rebalance will be further delayed by the value of group.initial.rebalance.delay.ms as new members join the group, up to a maximum of max.poll.interval.ms.
+# The default value for this is 3 seconds.
+# We override this to 0 here as it makes for a better out-of-the-box experience for development and testing.
+# However, in production environments the default value of 3 seconds is more suitable as this will help to avoid unnecessary, and potentially expensive, rebalances during application startup.
+group.initial.rebalance.delay.ms={{ grp_initial_rebalance_delay }}
+
+
+############################# Confluent Authorizer Settings #############################
+
+# Uncomment to enable Confluent Authorizer with support for ACLs, LDAP groups and RBAC
+#authorizer.class.name=io.confluent.kafka.security.authorizer.ConfluentServerAuthorizer
+# Semi-colon separated list of super users in the format <principalType>:<principalName>
+#super.users=
+# Specify a valid Confluent license. By default free-tier license will be used
+#confluent.license=
+# Replication factor for the topic used for licensing. Default is 3.
+confluent.license.topic.replication.factor=1
+
+# Uncomment the following lines and specify values where required to enable RBAC
+# Enable RBAC provider
+#confluent.authorizer.access.rule.providers=ACL,RBAC
+# Bootstrap servers for RBAC metadata. Must be provided if this broker is not in the metadata cluster
+#confluent.metadata.bootstrap.servers=PLAINTEXT://127.0.0.1:9092
+# Replication factor for the metadata topic used for authorization. Default is 3.
+confluent.metadata.topic.replication.factor=1
+
+# Listeners for metadata server
+#confluent.metadata.server.listeners=http://0.0.0.0:8090
+# Advertised listeners for metadata server
+#confluent.metadata.server.advertised.listeners=http://127.0.0.1:8090
diff --git a/ansible/roles/mft/tasks/main.yml b/ansible/roles/mft/tasks/main.yml
index dc6e243..a382136 100644
--- a/ansible/roles/mft/tasks/main.yml
+++ b/ansible/roles/mft/tasks/main.yml
@@ -55,6 +55,20 @@
become: yes
become_user: "{{ user }}"
+- name: Delete lib directories of the distributions
+ file:
+ state: absent
+ path: "{{ mft_deployment_dir }}/{{ item }}"
+ with_items:
+ - "MFT-Controller-{{mft_distribution_version}}/lib"
+ - "MFT-Agent-{{mft_distribution_version}}/lib"
+ - "API-Service-{{mft_distribution_version}}/lib"
+ - "Resource-Service-{{mft_distribution_version}}/lib"
+ - "Secret-Service-{{mft_distribution_version}}/lib"
+ become: yes
+ become_user: "{{ user }}"
+ ignore_errors: yes
+
- name: Copy MFT distributions to MFT deployment directory
unarchive: "src={{ mft_source_dir }}/{{ item }}
dest={{ mft_deployment_dir }}/ copy=no"
diff --git a/ansible/roles/mft/templates/secret-service/application.properties.j2 b/ansible/roles/mft/templates/secret-service/application.properties.j2
index 332b2bb..0f3a14a 100644
--- a/ansible/roles/mft/templates/secret-service/application.properties.j2
+++ b/ansible/roles/mft/templates/secret-service/application.properties.j2
@@ -28,5 +28,7 @@ file.backend.secret.file=secrets.json
# Configurations for Custos Backend
custos.host=custos.scigap.org
custos.port=31499
-custos.id=CHANGE_ME
-custos.secret=CHANGE_ME
\ No newline at end of file
+custos.id={{ mft_secret_service_custos_client_id }}
+custos.secret={{ mft_secret_service_custos_client_secret }}
+custos.backend.drms.host={{ datalake_drms_host }}
+custos.backend.drms.port={{ datalake_drms_grpc_port }}
\ No newline at end of file
diff --git a/ansible/roles/kafka/defaults/main.yml b/ansible/roles/zookeeper/handlers/main.yml
similarity index 75%
copy from ansible/roles/kafka/defaults/main.yml
copy to ansible/roles/zookeeper/handlers/main.yml
index 2a095ce..a15d1b2 100644
--- a/ansible/roles/kafka/defaults/main.yml
+++ b/ansible/roles/zookeeper/handlers/main.yml
@@ -19,8 +19,14 @@
#
---
+- name: start zookeeper
+ service: name=zookeeper state=started enabled=yes
+ become: yes
-kafka_package_name: "confluent-5.3.1"
-kafka_tgz_url: "http://packages.confluent.io/archive/5.3/confluent-5.3.1-2.12.zip"
-kafka_dir: "{{ datalake_deployment_dir }}/{{ kafka_package_name }}"
-kafka_rest_proxy_listener_port: 8082
\ No newline at end of file
+- name: stop zookeeper
+ service: name=zookeeper state=stopped
+ become: yes
+
+- name: restart zookeeper
+ service: name=zookeeper state=restarted enabled=yes
+ become: yes
diff --git a/ansible/roles/zookeeper/tasks/main.yml b/ansible/roles/zookeeper/tasks/main.yml
new file mode 100644
index 0000000..4d83773
--- /dev/null
+++ b/ansible/roles/zookeeper/tasks/main.yml
@@ -0,0 +1,59 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+################################################################################
+# Setup and run Zookeeper
+- name: Download and unarchive zookeeper
+ unarchive: src="{{ zookeeper_url }}"
+ dest="{{ user_home }}"
+ copy=no
+ owner="{{ user }}"
+ group="{{ group }}"
+ become: yes
+
+- name: Copy zoo.cfg file
+ template: src=zoo.cfg.j2 dest="{{ zookeeper_dir }}/conf/zoo.cfg" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ notify:
+ - restart zookeeper
+ become: yes
+
+- name: Copy java.env file
+ template: src=java.env.j2 dest="{{ zookeeper_dir }}/conf/java.env" owner="{{ user }}" group="{{ group }}" mode="u=rw,g=r,o=r"
+ notify:
+ - restart zookeeper
+ become: yes
+
+- name: Check if systemd exists
+ stat: path=/usr/lib/systemd/system/
+ register: systemd_check
+ become: yes
+
+- name: Systemd script.
+ template: src=zookeeper.service.j2 dest=/usr/lib/systemd/system/zookeeper.service
+ when: systemd_check.stat.exists == true
+ notify:
+ - restart zookeeper
+ become: yes
+
+- name: reload systemd daemons
+ command: systemctl daemon-reload
+ become: yes
+...
diff --git a/ansible/roles/zookeeper/templates/java.env.j2 b/ansible/roles/zookeeper/templates/java.env.j2
new file mode 100644
index 0000000..356192c
--- /dev/null
+++ b/ansible/roles/zookeeper/templates/java.env.j2
@@ -0,0 +1 @@
+export ZOO_LOG_DIR={{ zookeeper_dir }}/logs
\ No newline at end of file
diff --git a/ansible/roles/zookeeper/templates/zoo.cfg.j2 b/ansible/roles/zookeeper/templates/zoo.cfg.j2
new file mode 100644
index 0000000..0c40776
--- /dev/null
+++ b/ansible/roles/zookeeper/templates/zoo.cfg.j2
@@ -0,0 +1,28 @@
+# The number of milliseconds of each tick
+tickTime={{tick_time}}
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit={{init_limit}}
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit={{sync_limit}}
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just
+# example sakes.
+dataDir={{zookeeper_data_dir}}
+# the port at which the clients will connect
+clientPort={{ client_port }}
+# the maximum number of client connections.
+# increase this if you need to handle more clients
+#maxClientCnxns=60
+#
+# Be sure to read the maintenance section of the
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to "0" to disable auto purge feature
+autopurge.purgeInterval=1
diff --git a/ansible/roles/zookeeper/templates/zookeeper.service.j2 b/ansible/roles/zookeeper/templates/zookeeper.service.j2
new file mode 100644
index 0000000..8e6ea2a
--- /dev/null
+++ b/ansible/roles/zookeeper/templates/zookeeper.service.j2
@@ -0,0 +1,16 @@
+# {{ansible_managed}}
+
+[Unit]
+Description=ZooKeeper
+
+
+[Service]
+Type=forking
+ExecStart={{zookeeper_dir}}/bin/zkServer.sh start
+ExecStop={{ zookeeper_dir }}/bin/zkServer.sh stop
+Restart=always
+TimeoutSec=300
+User={{ user }}
+
+[Install]
+WantedBy=multi-user.target
diff --git a/ansible/roles/kafka/handlers/main.yml b/ansible/roles/zookeeper/vars/main.yml
similarity index 61%
copy from ansible/roles/kafka/handlers/main.yml
copy to ansible/roles/zookeeper/vars/main.yml
index 761e265..6171334 100644
--- a/ansible/roles/kafka/handlers/main.yml
+++ b/ansible/roles/zookeeper/vars/main.yml
@@ -19,15 +19,19 @@
#
---
-# Kafka related handlers
-- name: start kafka-rest-proxy
- service: name=kafka-rest-proxy state=started enabled=yes
- become: yes
+#Variables associated with this role
+# zookeeper related variable
+zookeeper_version: 3.4.8
+zookeeper_url: http://archive.apache.org/dist/zookeeper/zookeeper-{{zookeeper_version}}/zookeeper-{{zookeeper_version}}.tar.gz
-- name: stop kafka-rest-proxy
- service: name=kafka-rest-proxy state=stopped enabled=yes
- become: yes
+apt_cache_timeout: 3600
+client_port: "{{ zookeeper_client_port }}"
+init_limit: 5
+sync_limit: 2
+tick_time: 2000
+data_dir: /var/lib/zookeeper
+log_dir: /var/log/zookeeper
+zookeeper_dir: "{{ user_home }}/zookeeper-{{zookeeper_version}}"
+zookeeper_data_dir: "{{ zookeeper_dir }}/data"
-- name: restart kafka-rest-proxy
- service: name=kafka-rest-proxy state=restarted enabled=yes
- become: yes
+...
diff --git a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/OrchClient.java b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/OrchClient.java
index 2890dfe..09c512c 100644
--- a/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/OrchClient.java
+++ b/data-orchestrator/data-orchestrator-service/data-orchestrator-api-server/src/main/java/org/apache/airavata/datalake/orchestrator/OrchClient.java
@@ -25,12 +25,14 @@ public class OrchClient {
mapper.map(dpe, builder);
System.out.println(builder.build());*/
- ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 6565).usePlaintext().build();
+ ManagedChannel channel = ManagedChannelBuilder.forAddress("localhost", 6566).usePlaintext().build();
DataParserServiceGrpc.DataParserServiceBlockingStub parserClient = DataParserServiceGrpc.newBlockingStub(channel);
- ParsingJobListResponse parsingJobListResponse = parserClient.listParsingJobs(ParsingJobListRequest.newBuilder().build());
- System.out.println(parsingJobListResponse);
- /*String parserId = UUID.randomUUID().toString();
+ //ParsingJobListResponse parsingJobListResponse = parserClient.listParsingJobs(ParsingJobListRequest.newBuilder().build());
+ //System.out.println(parsingJobListResponse);
+ String parserId = UUID.randomUUID().toString();
+ String dm3InputId = UUID.randomUUID().toString();
+ String jsonOutputId = UUID.randomUUID().toString();
ParserRegisterResponse parserRegisterResponse = parserClient.registerParser(ParserRegisterRequest.newBuilder()
.setParser(DataParser.newBuilder()
.setParserId(parserId)
@@ -41,12 +43,12 @@ public class OrchClient {
.setOutputPath("/opt/outputs")
.addInputInterfaces(DataParserInputInterface.newBuilder()
.setParserId(parserId)
- .setParserInputInterfaceId(UUID.randomUUID().toString())
+ .setParserInputInterfaceId(dm3InputId)
.setInputType("FILE")
.setInputName("input.dm3").build())
.addOutputInterfaces(DataParserOutputInterface.newBuilder()
.setParserId(parserId)
- .setParserOutputInterfaceId(UUID.randomUUID().toString())
+ .setParserOutputInterfaceId(jsonOutputId)
.setOutputType("FILE")
.setOutputName("derrived-metadata.json")
.build())
@@ -66,19 +68,20 @@ public class OrchClient {
String parsingJobId = UUID.randomUUID().toString();
ParsingJobRegisterRequest.Builder parsingJob = ParsingJobRegisterRequest.newBuilder().setParsingJob(DataParsingJob.newBuilder()
.setDataParsingJobId(parsingJobId)
- .setParserId("2f48d838-d0e4-4413-8d8e-4579d4a7369d")
+ .setParserId(parserId)
.addDataParsingJobInputs(DataParsingJobInput.newBuilder()
.setId(UUID.randomUUID().toString())
- .setDataParserInputInterfaceId("dbb221fc-96d1-4aaf-ae85-73d6215a5106")
+ .setDataParserInputInterfaceId(dm3InputId)
.setDataParsingJobId(parsingJobId)
.setSelectionQuery("EXTENSION == dm3").build())
.addDataParsingJobOutputs(DataParsingJobOutput.newBuilder()
.setId(UUID.randomUUID().toString())
.setDataParsingJobId(parsingJobId)
.setOutputType("JSON")
- .setDataParserOutputInterfaceId("d1ded2cc-ffe6-4e47-a5a2-bdb782b62359").build()).build());
- ParsingJobRegisterResponse parsingJobRegisterResponse = parserClient.registerParsingJob(parsingJob.build()); */
+ .setDataParserOutputInterfaceId(jsonOutputId).build()).build());
+ ParsingJobRegisterResponse parsingJobRegisterResponse = parserClient.registerParsingJob(parsingJob.build());
+ System.out.println(parsingJobRegisterResponse);
/*File f = new File("/tmp/a.dm3");
ScriptEngine engine = new ScriptEngineManager().getEngineByName("JavaScript");
@@ -86,6 +89,6 @@ public class OrchClient {
bindings.put("polyglot.js.allowHostAccess", true);
bindings.put("polyglot.js.allowHostClassLookup", (Predicate<String>) s -> true);
bindings.put("inputFile", f);
- System.out.println(engine.eval("inputFile.getName().endsWith(\".dm3\");")); // it will not work without allowHostAccess and allowHostClassLookup*/
+ System.out.println(engine.eval("metadata.getFriendlyName().endsWith(\".dm3\");")); // it will not work without allowHostAccess and allowHostClassLookup*/
}
}