You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/03/13 02:20:48 UTC
[inlong] branch master updated: [INLONG-7572][Docker] Add the Kafka support for audit docker deployment (#7573)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 715e1a226 [INLONG-7572][Docker] Add the Kafka support for audit docker deployment (#7573)
715e1a226 is described below
commit 715e1a22640896f5d4dce65e6d2b630769bac86f
Author: Charles Zhang <do...@apache.org>
AuthorDate: Mon Mar 13 10:20:42 2023 +0800
[INLONG-7572][Docker] Add the Kafka support for audit docker deployment (#7573)
---
docker/docker-compose/docker-compose.yml | 6 +++++-
docker/kubernetes/templates/audit-statefulset.yaml | 6 ++++++
docker/kubernetes/values.yaml | 3 +++
inlong-audit/audit-docker/Dockerfile | 2 +-
inlong-audit/audit-docker/audit-docker.sh | 11 +++++++++++
inlong-dataproxy/dataproxy-docker/Dockerfile | 2 +-
6 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/docker/docker-compose/docker-compose.yml b/docker/docker-compose/docker-compose.yml
index 9365b15bb..800d4415c 100644
--- a/docker/docker-compose/docker-compose.yml
+++ b/docker/docker-compose/docker-compose.yml
@@ -83,6 +83,7 @@ services:
- MANAGER_OPENAPI_IP=manager
- MANAGER_OPENAPI_PORT=8083
- AUDIT_PROXY_URL=audit:10081
+ # pulsar or kafka
- MQ_TYPE=pulsar
- ETH_NAME=eth0
@@ -113,8 +114,11 @@ services:
- JDBC_URL=mysql:3306
- USERNAME=root
- PASSWORD=inlong
- - PULSAR_BROKER_LIST=pulsar:6650
+ # pulsar or kafka
- MQ_TYPE=pulsar
+ - PULSAR_BROKER_LIST=pulsar:6650
+ # for kafka
+ # - KAFKA_SERVER_LIST=kafka:9092
ports:
- "10081:10081"
diff --git a/docker/kubernetes/templates/audit-statefulset.yaml b/docker/kubernetes/templates/audit-statefulset.yaml
index 5a1a0b40a..c2f557fd9 100644
--- a/docker/kubernetes/templates/audit-statefulset.yaml
+++ b/docker/kubernetes/templates/audit-statefulset.yaml
@@ -124,6 +124,12 @@ spec:
- name: PULSAR_AUDIT_TOPIC
value: "{{ .Values.external.pulsar.auditTopicName }}"
{{- end }}
+ {{ if eq .Values.dataproxy.mqType "kafka" }}
+ - name: MQ_TYPE
+ value: "kafka"
+ - name: KAFKA_SERVER_LIST
+ value: {{ .Values.external.kafka.bootstrap_server_url }}
+ {{- end }}
{{ if eq .Values.dataproxy.mqType "tubemq" }}
- name: MQ_TYPE
value: "tubemq"
diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml
index ca41b97aa..b6e8eec58 100644
--- a/docker/kubernetes/values.yaml
+++ b/docker/kubernetes/values.yaml
@@ -670,6 +670,9 @@ external:
serviceUrl: "localhost:6650"
adminUrl: "localhost:8080"
auditTopicName: "persistent://public/default/inlong-audit"
+ # If choose Kafka as an MQ service for DataProxy, you need to configure the cluster address.
+ kafka:
+ bootstrap_server_url: "localhost:9092"
# the REST server address for Flink
flink:
hostname: "127.0.0.1"
diff --git a/inlong-audit/audit-docker/Dockerfile b/inlong-audit/audit-docker/Dockerfile
index 1c5edd7b3..88939e2b3 100644
--- a/inlong-audit/audit-docker/Dockerfile
+++ b/inlong-audit/audit-docker/Dockerfile
@@ -24,7 +24,7 @@ RUN apt-get update \
ARG AUDIT_TARBALL
ADD ${AUDIT_TARBALL} /opt/inlong-audit
EXPOSE 10081
-# pulsar or tubemq
+# pulsar or tubemq or kafka
ENV MQ_TYPE=pulsar
ENV PULSAR_BROKER_LIST="127.0.0.1:6650"
ENV PULSAR_AUDIT_TOPIC="persistent:\/\/public\/default\/inlong-audit"
diff --git a/inlong-audit/audit-docker/audit-docker.sh b/inlong-audit/audit-docker/audit-docker.sh
index 5f05e8235..fdc367ca7 100755
--- a/inlong-audit/audit-docker/audit-docker.sh
+++ b/inlong-audit/audit-docker/audit-docker.sh
@@ -26,6 +26,7 @@ sql_ck_file="${file_path}"/sql/apache_inlong_audit_clickhouse.sql
# replace the configuration for audit proxy
if [ "${MQ_TYPE}" = "pulsar" ]; then
+ sed -i "s/audit.config.proxy.type=.*$/audit.config.proxy.type=pulsar"/g "${store_conf_file}"
sed -i "s/audit.pulsar.server.url=.*$/audit.pulsar.server.url=pulsar:\/\/${PULSAR_BROKER_LIST}/g" "${store_conf_file}"
sed -i "s/audit.pulsar.topic = .*$/audit.pulsar.topic = ${PULSAR_AUDIT_TOPIC}/g" "${store_conf_file}"
sed -i "s/agent1.sinks.pulsar-sink-msg1.pulsar_server_url = .*$/agent1.sinks.pulsar-sink-msg1.pulsar_server_url = pulsar:\/\/${PULSAR_BROKER_LIST}/g" "${proxy_conf_file}"
@@ -33,7 +34,14 @@ if [ "${MQ_TYPE}" = "pulsar" ]; then
sed -i "s/agent1.sinks.pulsar-sink-msg1.topic = .*$/agent1.sinks.pulsar-sink-msg1.topic = ${PULSAR_AUDIT_TOPIC}/g" "${proxy_conf_file}"
sed -i "s/agent1.sinks.pulsar-sink-msg2.topic = .*$/agent1.sinks.pulsar-sink-msg2.topic = ${PULSAR_AUDIT_TOPIC}/g" "${proxy_conf_file}"
fi
+if [ "${MQ_TYPE}" = "kafka" ]; then
+ sed -i "s/audit.config.proxy.type=.*$/audit.config.proxy.type=kafka"/g "${store_conf_file}"
+ sed -i "s/audit.kafka.server.url=.*$/audit.kafka.server.url=${KAFKA_SERVER_LIST}"/g "${store_conf_file}"
+ sed -i "s/agent1.sinks.kafka-sink-msg1.bootstrap_servers = .*$/agent1.sinks.kafka-sink-msg1.bootstrap_servers = ${KAFKA_SERVER_LIST}/g" "${proxy_conf_file}"
+ sed -i "s/agent1.sinks.kafka-sink-msg2.bootstrap_servers = .*$/agent1.sinks.kafka-sink-msg2.bootstrap_servers = ${KAFKA_SERVER_LIST}/g" "${proxy_conf_file}"
+fi
if [ "${MQ_TYPE}" = "tubemq" ]; then
+ sed -i "s/audit.config.proxy.type=.*$/audit.config.proxy.type=tube"/g "${store_conf_file}"
sed -i "s/audit.tube.masterlist=.*$/audit.tube.masterlist=${TUBE_MASTER_LIST}/g" "${store_conf_file}"
sed -i "s/audit.tube.topic = .*$/audit.tube.topic = ${TUBE_AUDIT_TOPIC}/g" "${store_conf_file}"
sed -i "s/agent1.sinks.tube-sink-msg1.master-host-port-list = .*$/agent1.sinks.tube-sink-msg1.master-host-port-list = ${TUBE_MASTER_LIST}/g" "${proxy_conf_file}"
@@ -84,6 +92,9 @@ if [ "${START_MODE}" = "all" ] || [ "${START_MODE}" = "proxy" ]; then
if [ "${MQ_TYPE}" = "pulsar" ]; then
bash +x ./bin/proxy-start.sh pulsar
fi
+ if [ "${MQ_TYPE}" = "kafka" ]; then
+ bash +x ./bin/proxy-start.sh kafka
+ fi
if [ "${MQ_TYPE}" = "tubemq" ]; then
bash +x ./bin/proxy-start.sh tubemq
fi
diff --git a/inlong-dataproxy/dataproxy-docker/Dockerfile b/inlong-dataproxy/dataproxy-docker/Dockerfile
index 11b314338..9e5e9f8a6 100644
--- a/inlong-dataproxy/dataproxy-docker/Dockerfile
+++ b/inlong-dataproxy/dataproxy-docker/Dockerfile
@@ -31,7 +31,7 @@ ENV ETH_NAME=eth0
# enable audit, true or false
ENV AUDIT_ENABLE=true
ENV AUDIT_PROXY_URL=127.0.0.1:10081
-# pulsar or tubemq
+# pulsar or tubemq or kafka
ENV MQ_TYPE=pulsar
ENV CLUSTER_TAG=default_cluster
ENV CLUSTER_NAME=default_dataproxy