You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by di...@apache.org on 2021/09/01 17:37:06 UTC
[airavata-data-lake] branch master updated: Adding kafka rest proxy
deployment
This is an automated email from the ASF dual-hosted git repository.
dimuthuupe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airavata-data-lake.git
The following commit(s) were added to refs/heads/master by this push:
new 5c23e0e Adding kafka rest proxy deployment
5c23e0e is described below
commit 5c23e0eab5213a210dacbcfa11f60c46d46ac464
Author: Dimuthu Wannipurage <di...@gmail.com>
AuthorDate: Wed Sep 1 13:36:59 2021 -0400
Adding kafka rest proxy deployment
---
ansible/datalake.yml | 4 +-
ansible/inventories/test/group_vars/all/vars.yml | 9 +--
.../kafka/defaults/main.yml} | 12 ++--
.../kafka/handlers/main.yml} | 19 ++++--
ansible/roles/kafka/tasks/main.yml | 79 ++++++++++++++++++++++
.../kafka/templates/kafka-rest-proxy.service.j2 | 14 ++++
.../roles/kafka/templates/kafka-rest.properties.j2 | 25 +++++++
7 files changed, 143 insertions(+), 19 deletions(-)
diff --git a/ansible/datalake.yml b/ansible/datalake.yml
index dbfc529..273a548 100644
--- a/ansible/datalake.yml
+++ b/ansible/datalake.yml
@@ -21,8 +21,10 @@
---
- hosts: data-lake
roles:
+ - env_setup
+ - kafka
# - custos
# - consul
# - mft
- - data_lake
+# - data_lake
# - workflow-engine
diff --git a/ansible/inventories/test/group_vars/all/vars.yml b/ansible/inventories/test/group_vars/all/vars.yml
index 134537b..48ec095 100644
--- a/ansible/inventories/test/group_vars/all/vars.yml
+++ b/ansible/inventories/test/group_vars/all/vars.yml
@@ -30,6 +30,7 @@ datalake_deployment_dir: "{{ user_home }}/data-lake-deployment"
datalake_source_dir: "{{ user_home }}/data-lake-source"
datalake_repo: "https://github.com/apache/airavata-data-lake.git"
datalake_git_branch: master
+kafka_url: 149.165.156.200:9092
mft_deployment_dir: "{{ user_home }}/mft-deployment"
mft_source_dir: "{{ user_home }}/mft-source"
@@ -84,7 +85,7 @@ datalake_drms_custos_host: custos.scigap.org
datalake_drms_custos_port: 31499
datalake_drms_http_port: 8900
-datalake_synch_broker_url: 149.165.156.200:9092
+datalake_synch_broker_url: "{{ kafka_url }}"
datalake_synch_broker_consumer_group: custosEventsGroup
datalake_synch_broker_topic: 10002640-561b67b4-fa7d-4368-8643-dcd9d0159c1d
datalake_synch_tenant_id: custos-ii8g0cfwsz6ruwezykn9-10002640
@@ -94,8 +95,8 @@ datalake_data_orch_datasource_username: "{{ vault_datalake_data_orch_datasource_
datalake_data_orch_datasource_password: "{{ vault_datalake_data_orch_datasource_password }}"
datalake_data_orch_http_port: 8899
datalake_data_orch_grpc_port: 6566
-datalake_data_orch_broker_url: localhost:9092
-datalake_data_orch_broker_consumer_group: DataOrchestratorGroup3
+datalake_data_orch_broker_url: "{{ kafka_url }}"
+datalake_data_orch_broker_consumer_group: DataOrchestratorGroup2
datalake_data_orch_broker_topic: data-orchestrator-file-events
workflow_deployment_dir: "{{ user_home }}/workflow-deployment"
@@ -105,7 +106,7 @@ workflow_helix_cluster_name: datalake
workflow_datasync_wm_name: datasync_wf
workflow_parsing_wm_name: parsing_wf
-workflow_worker_kafka_url: 149.165.156.200:9092
+workflow_worker_kafka_url: "{{ kafka_url }}"
workflow_worker_kafka_publisher_name: mft-status-publisher
workflow_worker_kafka_publish_topic: mft-status-topic
workflow_worker_kafka_consumner_group: mft-even-group
diff --git a/ansible/datalake.yml b/ansible/roles/kafka/defaults/main.yml
similarity index 77%
copy from ansible/datalake.yml
copy to ansible/roles/kafka/defaults/main.yml
index dbfc529..2a095ce 100644
--- a/ansible/datalake.yml
+++ b/ansible/roles/kafka/defaults/main.yml
@@ -19,10 +19,8 @@
#
---
-- hosts: data-lake
- roles:
-# - custos
-# - consul
-# - mft
- - data_lake
-# - workflow-engine
+
+kafka_package_name: "confluent-5.3.1"
+kafka_tgz_url: "http://packages.confluent.io/archive/5.3/confluent-5.3.1-2.12.zip"
+kafka_dir: "{{ datalake_deployment_dir }}/{{ kafka_package_name }}"
+kafka_rest_proxy_listener_port: 8082
\ No newline at end of file
diff --git a/ansible/datalake.yml b/ansible/roles/kafka/handlers/main.yml
similarity index 69%
copy from ansible/datalake.yml
copy to ansible/roles/kafka/handlers/main.yml
index dbfc529..761e265 100644
--- a/ansible/datalake.yml
+++ b/ansible/roles/kafka/handlers/main.yml
@@ -19,10 +19,15 @@
#
---
-- hosts: data-lake
- roles:
-# - custos
-# - consul
-# - mft
- - data_lake
-# - workflow-engine
+# Kafka related handlers
+- name: start kafka-rest-proxy
+ service: name=kafka-rest-proxy state=started enabled=yes
+ become: yes
+
+- name: stop kafka-rest-proxy
+ service: name=kafka-rest-proxy state=stopped enabled=yes
+ become: yes
+
+- name: restart kafka-rest-proxy
+ service: name=kafka-rest-proxy state=restarted enabled=yes
+ become: yes
diff --git a/ansible/roles/kafka/tasks/main.yml b/ansible/roles/kafka/tasks/main.yml
new file mode 100644
index 0000000..40360d6
--- /dev/null
+++ b/ansible/roles/kafka/tasks/main.yml
@@ -0,0 +1,79 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+
+# Check for the availability of the Kafka package
+- name: Check kafka package availability
+ stat: path={{ kafka_dir }}
+ register: kafka_package
+ become: yes
+
+- name: Create deployment directory {{ datalake_deployment_dir }}
+ file: path={{datalake_deployment_dir}}
+ state=directory
+ mode=0755
+ owner={{ user }}
+ group={{ group }}
+ become: yes
+
+# Download Kafka
+- name: Download and unarchive Kafka from {{ kafka_tgz_url }}
+ unarchive: src="{{ kafka_tgz_url }}"
+ dest="{{ datalake_deployment_dir }}"
+ copy=no
+ owner="{{ user }}"
+ group="{{ group }}"
+ when: not kafka_package.stat.exists
+ become: yes
+
+# Create kafka logs directory
+- name: Create kafka logs directory
+ file: path="{{ kafka_dir }}/logs" state=directory owner={{ user }} group={{ group }}
+ become: yes
+
+# Config rest proxy and start
+- name: Copy kafka rest proxy properties file
+ template: src=kafka-rest.properties.j2
+ dest="{{ kafka_dir }}/etc/kafka-rest/kafka-rest.properties"
+ owner={{ user }}
+ group={{ group }}
+ mode="u=rw,g=r,o=r"
+ become: yes
+
+- name: open kafka rest proxy port
+ firewalld: port="{{ kafka_rest_proxy_listener_port }}/tcp"
+ zone=public permanent=true state=enabled immediate=yes
+ become: yes
+
+- name: systemd install kafka rest proxy service script
+ template: src=kafka-rest-proxy.service.j2
+ dest=/usr/lib/systemd/system/kafka-rest-proxy.service
+ owner={{ user }}
+ group={{ group }}
+ mode="u=rw,g=r,o=r"
+ become: yes
+
+- name: Reload systemd daemons
+ command: systemctl daemon-reload
+ notify: restart kafka-rest-proxy
+ become: yes
+
+...
diff --git a/ansible/roles/kafka/templates/kafka-rest-proxy.service.j2 b/ansible/roles/kafka/templates/kafka-rest-proxy.service.j2
new file mode 100644
index 0000000..338de9f
--- /dev/null
+++ b/ansible/roles/kafka/templates/kafka-rest-proxy.service.j2
@@ -0,0 +1,14 @@
+# {{ansible_managed}}
+
+[Unit]
+Description=Kafka-Rest-Proxy
+Before=
+After=network.target
+
+[Service]
+LOG_DIR={{ kafka_dir }}/logs
+ExecStart={{ kafka_dir }}/bin/kafka-rest-start {{ kafka_dir }}/etc/kafka-rest/kafka-rest.properties
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target
\ No newline at end of file
diff --git a/ansible/roles/kafka/templates/kafka-rest.properties.j2 b/ansible/roles/kafka/templates/kafka-rest.properties.j2
new file mode 100644
index 0000000..a512daa
--- /dev/null
+++ b/ansible/roles/kafka/templates/kafka-rest.properties.j2
@@ -0,0 +1,25 @@
+#
+# Copyright 2018 Confluent Inc.
+#
+# Licensed under the Confluent Community License (the "License"); you may not use
+# this file except in compliance with the License. You may obtain a copy of the
+# License at
+#
+# http://www.confluent.io/confluent-community-license
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+
+#id=kafka-rest-test-server
+#schema.registry.url=http://localhost:8081
+#zookeeper.connect=localhost:2181
+bootstrap.servers=PLAINTEXT://{{ kafka_url }}
+listeners=http://{{ ansible_default_ipv4.address }}:{{ kafka_rest_proxy_listener_port }}
+#
+# Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center
+# Make sure that monitoring-interceptors-<version>.jar is on the Java class path
+#consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
+#producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor