You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by do...@apache.org on 2023/02/28 01:48:37 UTC
[inlong] branch master updated: [INLONG-7447][Docker] Make the Pulsar the default MQ option to lower the complexity (#7448)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 6fb7873da [INLONG-7447][Docker] Make the Pulsar the default MQ option to lower the complexity (#7448)
6fb7873da is described below
commit 6fb7873da4330d063b459aacd7e474661450dee9
Author: Charles Zhang <do...@apache.org>
AuthorDate: Tue Feb 28 09:48:31 2023 +0800
[INLONG-7447][Docker] Make the Pulsar the default MQ option to lower the complexity (#7448)
---
docker/kubernetes/README.md | 7 ++++++-
docker/kubernetes/templates/audit-statefulset.yaml | 18 ++++++++++++++----
.../kubernetes/templates/tubemq-broker-configmap.yaml | 2 +-
.../templates/tubemq-broker-ini-configmap.yaml | 2 +-
docker/kubernetes/templates/tubemq-broker-pdb.yaml | 2 +-
docker/kubernetes/templates/tubemq-broker-service.yaml | 2 +-
.../templates/tubemq-broker-statefulset.yaml | 2 +-
.../kubernetes/templates/tubemq-manager-service.yaml | 2 +-
.../templates/tubemq-manager-statefulset.yaml | 2 +-
.../kubernetes/templates/tubemq-master-configmap.yaml | 2 +-
.../templates/tubemq-master-ini-configmap.yaml | 2 +-
docker/kubernetes/templates/tubemq-master-pdb.yaml | 2 +-
docker/kubernetes/templates/tubemq-master-service.yaml | 2 +-
.../templates/tubemq-master-statefulset.yaml | 2 +-
docker/kubernetes/templates/zookeeper-configmap.yaml | 3 ++-
docker/kubernetes/templates/zookeeper-pdb.yaml | 3 ++-
docker/kubernetes/templates/zookeeper-service.yaml | 3 ++-
docker/kubernetes/templates/zookeeper-statefulset.yaml | 3 ++-
docker/kubernetes/values.yaml | 17 ++++++++---------
19 files changed, 48 insertions(+), 30 deletions(-)
diff --git a/docker/kubernetes/README.md b/docker/kubernetes/README.md
index e877a370e..3d946a973 100644
--- a/docker/kubernetes/README.md
+++ b/docker/kubernetes/README.md
@@ -4,10 +4,16 @@
- Kubernetes 1.10+
- Helm 3.0+
+- [Flink 1.13.5](https://nightlies.apache.org/flink/flink-docs-release-1.13/docs/try-flink/local_installation/)
+- [Apache Pulsar 2.8.x](https://pulsar.apache.org/docs/en/2.8.1/standalone/)
- A dynamic provisioner for the PersistentVolumes(`production environment`)
## Usage
+### Configure
+- configure the `external.pulsar` for Apache Pulsar info in `values.yaml`
+- configure the `external.flink` for Apache Flink info in `values.yaml`
+
### Install
If the namespace named `inlong` does not exist, create it first by running:
@@ -98,7 +104,6 @@ The configuration file is [values.yaml](values.yaml), and the following tables l
| `external.mysql.port` | `3306` | External MySQL port |
| `external.mysql.username` | `root` | External MySQL username |
| `external.mysql.password` | `password` | External MySQL password |
-| `external.pulsar.enabled` | `false` | If not exists external Pulsar, InLong will use the internal TubeMQ by default |
| `external.pulsar.serviceUrl` | `localhost:6650` | External Pulsar service URL |
| `external.pulsar.adminUrl` | `localhost:8080` | External Pulsar admin URL |
diff --git a/docker/kubernetes/templates/audit-statefulset.yaml b/docker/kubernetes/templates/audit-statefulset.yaml
index ef2be5105..5a1a0b40a 100644
--- a/docker/kubernetes/templates/audit-statefulset.yaml
+++ b/docker/kubernetes/templates/audit-statefulset.yaml
@@ -116,14 +116,24 @@ spec:
secretKeyRef:
name: {{ template "inlong.fullname" . }}-{{ .Values.mysql.component }}
key: mysql-password
+ {{ if eq .Values.dataproxy.mqType "pulsar" }}
+ - name: MQ_TYPE
+ value: "pulsar"
+ - name: PULSAR_BROKER_LIST
+ value: {{ .Values.external.pulsar.serviceUrl }}
+ - name: PULSAR_AUDIT_TOPIC
+ value: "{{ .Values.external.pulsar.auditTopicName }}"
+ {{- end }}
+ {{ if eq .Values.dataproxy.mqType "tubemq" }}
+ - name: MQ_TYPE
+ value: "tubemq"
+ - name: TUBE_MASTER_LIST
+ value: "{{ template "inlong.tubemqMaster.hostname" . }}:{{ .Values.tubemqMaster.ports.rpcPort }}"
+ {{- end }}
- name: MANAGER_OPENAPI_IP
value: {{ include "inlong.manager.hostname" . | quote }}
- name: MANAGER_OPENAPI_PORT
value: {{ .Values.manager.port | quote }}
- - name: PULSAR_BROKER_LIST
- value: {{ .Values.external.pulsar.serviceUrl | quote }}
- - name: TUBE_MASTER_LIST
- value: "{{ template "inlong.tubemqMaster.hostname" . }}:{{ .Values.tubemqMaster.ports.rpcPort }}"
{{- range $key, $value := .Values.audit.env }}
- name: {{ $key }}
value: {{ $value | quote }}
diff --git a/docker/kubernetes/templates/tubemq-broker-configmap.yaml b/docker/kubernetes/templates/tubemq-broker-configmap.yaml
index 84e39426d..97af397fc 100644
--- a/docker/kubernetes/templates/tubemq-broker-configmap.yaml
+++ b/docker/kubernetes/templates/tubemq-broker-configmap.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: ConfigMap
metadata:
diff --git a/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml b/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml
index aff65fe4b..df3e86c1f 100644
--- a/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml
+++ b/docker/kubernetes/templates/tubemq-broker-ini-configmap.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: ConfigMap
metadata:
diff --git a/docker/kubernetes/templates/tubemq-broker-pdb.yaml b/docker/kubernetes/templates/tubemq-broker-pdb.yaml
index 7b41d0eb9..225e63b3b 100644
--- a/docker/kubernetes/templates/tubemq-broker-pdb.yaml
+++ b/docker/kubernetes/templates/tubemq-broker-pdb.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
{{- if .Values.tubemqBroker.pdb.usePolicy }}
{{- if .Capabilities.APIVersions.Has "policy/v1" }}
apiVersion: policy/v1
diff --git a/docker/kubernetes/templates/tubemq-broker-service.yaml b/docker/kubernetes/templates/tubemq-broker-service.yaml
index 785e9a155..e466be6e5 100644
--- a/docker/kubernetes/templates/tubemq-broker-service.yaml
+++ b/docker/kubernetes/templates/tubemq-broker-service.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: Service
metadata:
diff --git a/docker/kubernetes/templates/tubemq-broker-statefulset.yaml b/docker/kubernetes/templates/tubemq-broker-statefulset.yaml
index 0666815fd..7201b18cd 100644
--- a/docker/kubernetes/templates/tubemq-broker-statefulset.yaml
+++ b/docker/kubernetes/templates/tubemq-broker-statefulset.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
diff --git a/docker/kubernetes/templates/tubemq-manager-service.yaml b/docker/kubernetes/templates/tubemq-manager-service.yaml
index b284bf7ce..e02f6840f 100644
--- a/docker/kubernetes/templates/tubemq-manager-service.yaml
+++ b/docker/kubernetes/templates/tubemq-manager-service.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqManager }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: Service
metadata:
diff --git a/docker/kubernetes/templates/tubemq-manager-statefulset.yaml b/docker/kubernetes/templates/tubemq-manager-statefulset.yaml
index 2042622fc..bb26f81f9 100644
--- a/docker/kubernetes/templates/tubemq-manager-statefulset.yaml
+++ b/docker/kubernetes/templates/tubemq-manager-statefulset.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqManager }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
diff --git a/docker/kubernetes/templates/tubemq-master-configmap.yaml b/docker/kubernetes/templates/tubemq-master-configmap.yaml
index 644e76de3..ac804bc5d 100644
--- a/docker/kubernetes/templates/tubemq-master-configmap.yaml
+++ b/docker/kubernetes/templates/tubemq-master-configmap.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: ConfigMap
metadata:
diff --git a/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml b/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml
index 019d8da7b..86ad1f4d4 100644
--- a/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml
+++ b/docker/kubernetes/templates/tubemq-master-ini-configmap.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: ConfigMap
metadata:
diff --git a/docker/kubernetes/templates/tubemq-master-pdb.yaml b/docker/kubernetes/templates/tubemq-master-pdb.yaml
index 94f145a72..4e19a16cd 100644
--- a/docker/kubernetes/templates/tubemq-master-pdb.yaml
+++ b/docker/kubernetes/templates/tubemq-master-pdb.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
{{- if .Values.tubemqMaster.pdb.usePolicy }}
{{- if .Capabilities.APIVersions.Has "policy/v1" }}
apiVersion: policy/v1
diff --git a/docker/kubernetes/templates/tubemq-master-service.yaml b/docker/kubernetes/templates/tubemq-master-service.yaml
index 26607c964..1bb4ae251 100644
--- a/docker/kubernetes/templates/tubemq-master-service.yaml
+++ b/docker/kubernetes/templates/tubemq-master-service.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: Service
metadata:
diff --git a/docker/kubernetes/templates/tubemq-master-statefulset.yaml b/docker/kubernetes/templates/tubemq-master-statefulset.yaml
index 63115563c..fd93b2b76 100644
--- a/docker/kubernetes/templates/tubemq-master-statefulset.yaml
+++ b/docker/kubernetes/templates/tubemq-master-statefulset.yaml
@@ -15,7 +15,7 @@
# limitations under the License.
#
-{{- if .Values.components.tubemqServer }}
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
diff --git a/docker/kubernetes/templates/zookeeper-configmap.yaml b/docker/kubernetes/templates/zookeeper-configmap.yaml
index 0452a5029..f8ffb77ea 100644
--- a/docker/kubernetes/templates/zookeeper-configmap.yaml
+++ b/docker/kubernetes/templates/zookeeper-configmap.yaml
@@ -15,7 +15,8 @@
# limitations under the License.
#
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: ConfigMap
metadata:
diff --git a/docker/kubernetes/templates/zookeeper-pdb.yaml b/docker/kubernetes/templates/zookeeper-pdb.yaml
index 083d65637..94dc9d7d1 100644
--- a/docker/kubernetes/templates/zookeeper-pdb.yaml
+++ b/docker/kubernetes/templates/zookeeper-pdb.yaml
@@ -15,7 +15,8 @@
# limitations under the License.
#
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
{{- if .Values.zookeeper.pdb.usePolicy }}
{{- if .Capabilities.APIVersions.Has "policy/v1" }}
apiVersion: policy/v1
diff --git a/docker/kubernetes/templates/zookeeper-service.yaml b/docker/kubernetes/templates/zookeeper-service.yaml
index 23352f373..6f6e006bf 100644
--- a/docker/kubernetes/templates/zookeeper-service.yaml
+++ b/docker/kubernetes/templates/zookeeper-service.yaml
@@ -15,7 +15,8 @@
# limitations under the License.
#
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: v1
kind: Service
metadata:
diff --git a/docker/kubernetes/templates/zookeeper-statefulset.yaml b/docker/kubernetes/templates/zookeeper-statefulset.yaml
index 4d08a5070..cad0c0ca4 100644
--- a/docker/kubernetes/templates/zookeeper-statefulset.yaml
+++ b/docker/kubernetes/templates/zookeeper-statefulset.yaml
@@ -15,7 +15,8 @@
# limitations under the License.
#
-{{- if .Values.components.zookeeper }}
+# only TubeMQ need zookeeper
+{{ if eq .Values.dataproxy.mqType "tubemq" }}
apiVersion: apps/v1
kind: StatefulSet
metadata:
diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml
index 2e13432fd..ca41b97aa 100644
--- a/docker/kubernetes/values.yaml
+++ b/docker/kubernetes/values.yaml
@@ -21,12 +21,9 @@ components:
agent: true
dashboard: true
dataproxy: true
- tubemqManager: true
- tubemqServer: true
manager: true
audit: true
mysql: true
- zookeeper: true
images:
agent:
@@ -59,7 +56,7 @@ images:
initContainer:
repository: busybox
tag: latest
- pullPolicy: "IfNotPresent"
+ pullPolicy: "Always"
ingress:
enabled: false
@@ -170,6 +167,8 @@ dashboard:
dataproxy:
component: "dataproxy"
+ # option, tubemq/pulsar/kafka, pulsar as default.
+ mqType: "pulsar"
replicas: 1
# The updateStrategy field allows you to configure and disable automated rolling updates for containers, labels, resource request/limits, and annotations for the Pods in a StatefulSet.
# There are two possible values: OnDelete and RollingUpdate.
@@ -658,20 +657,20 @@ tubemqBroker:
-XX:MaxRAMPercentage=80.0
-XX:-UseAdaptiveSizePolicy
-# If exists external MySQL or Pulsar, you can set the 'enable' field value to true and configure related information.
external:
+ # If exists external MySQL, you can set the 'enable' field value to true and configure related information.
mysql:
enabled: false
hostname: "localhost"
port: 3306
username: "root"
- password: "password"
- # If there is no external Pulsar, InLong will use TubeMQ.
+ password: "inlong"
+ # If choose Pulsar as an MQ service for DataProxy, you need to configure the cluster address.
pulsar:
- enabled: false
serviceUrl: "localhost:6650"
adminUrl: "localhost:8080"
+ auditTopicName: "persistent://public/default/inlong-audit"
+ # the REST server address for Flink
flink:
- enabled: true
hostname: "127.0.0.1"
port: 8081