You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@inlong.apache.org by al...@apache.org on 2022/02/21 07:04:46 UTC
[incubator-inlong] branch master updated: [INLONG-2510][Feature][Aduit][K8s] Add support for Audit on Kubernetes (#2619)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4f2690a [INLONG-2510][Feature][Aduit][K8s] Add support for Audit on Kubernetes (#2619)
4f2690a is described below
commit 4f2690a784acc8279376e7306b5f6b7ae8872307
Author: Yuanhao Ji <ts...@qq.com>
AuthorDate: Mon Feb 21 15:04:42 2022 +0800
[INLONG-2510][Feature][Aduit][K8s] Add support for Audit on Kubernetes (#2619)
* [INLONG-2510][Feature][Aduit][K8s] Add support for Audit on Kubernetes
issue: #2510
* [INLONG-2510][Feature][Aduit][K8s] Rebase and fix pulsar hostname
issue: #2510
* [INLONG-2510][Feature][Aduit][K8s] Fix mysql hostname on Audit
issue: #2510
* [INLONG-2510] Fix bugs when converting YAML to JSON
reference: https://stackoverflow.com/questions/62047714/error-converting-yaml-to-json-yaml-line-30-found-unknown-escape-character
issue: #2510
* [INLONG-2510] Fix init containers on Audit statefulset
issue: #2510
---
docker/kubernetes/templates/_helpers.tpl | 7 ++
docker/kubernetes/templates/audit-configmap.yaml | 28 +++++
docker/kubernetes/templates/audit-service.yaml | 33 ++++++
docker/kubernetes/templates/audit-statefulset.yaml | 122 +++++++++++++++++++++
docker/kubernetes/templates/pulsar-pvc.yaml | 36 ++++++
docker/kubernetes/templates/pulsar-service.yaml | 33 ++++++
.../kubernetes/templates/pulsar-statefulset.yaml | 66 +++++++++++
docker/kubernetes/values.yaml | 30 +++++
8 files changed, 355 insertions(+)
diff --git a/docker/kubernetes/templates/_helpers.tpl b/docker/kubernetes/templates/_helpers.tpl
index e3e90eb..808ecde 100644
--- a/docker/kubernetes/templates/_helpers.tpl
+++ b/docker/kubernetes/templates/_helpers.tpl
@@ -116,6 +116,13 @@ ${HOSTNAME}.{{ template "inlong.fullname" . }}-{{ .Values.zookeeper.component }}
{{- end -}}
{{/*
+Define the pulsar hostname
+*/}}
+{{- define "inlong.pulsar.hostname" -}}
+${HOSTNAME}.{{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}.{{ .Release.Namespace }}.svc.cluster.local
+{{- end -}}
+
+{{/*
Common labels
*/}}
{{- define "inlong.labels" -}}
diff --git a/docker/kubernetes/templates/audit-configmap.yaml b/docker/kubernetes/templates/audit-configmap.yaml
new file mode 100644
index 0000000..7f153ff
--- /dev/null
+++ b/docker/kubernetes/templates/audit-configmap.yaml
@@ -0,0 +1,28 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ namespace: {{ .Release.Namespace }}
+ labels:
+ component: {{ .Values.audit.component }}
+data:
+ mysqlUsername: {{ .Values.mysql.username | default "root" | quote }}
+ managerOpenApiPort: {{ .Values.manager.ports.webPort }}
+ pulsarBrokerUrlList: "pulsar://{{ include "inlong.pulsar.hostname" . }}:{{ .Values.pulsar.ports.pulsarPort }}"
diff --git a/docker/kubernetes/templates/audit-service.yaml b/docker/kubernetes/templates/audit-service.yaml
new file mode 100644
index 0000000..444c9ba
--- /dev/null
+++ b/docker/kubernetes/templates/audit-service.yaml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "inlong.commonLabels" . | nindent 4 }}
+ component: {{ .Values.audit.component }}
+spec:
+ ports:
+ - name: {{ .Values.audit.component}}-port
+ port: {{ .Values.audit.ports.webPort }}
+ type: ClusterIP
+ selector:
+ {{- include "inlong.matchLabels" . | nindent 4 }}
+ component: {{ .Values.audit.component }}
diff --git a/docker/kubernetes/templates/audit-statefulset.yaml b/docker/kubernetes/templates/audit-statefulset.yaml
new file mode 100644
index 0000000..33c0aff
--- /dev/null
+++ b/docker/kubernetes/templates/audit-statefulset.yaml
@@ -0,0 +1,122 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "inlong.commonLabels" . | nindent 4 }}
+ component: {{ .Values.audit.component }}
+spec:
+ serviceName: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ replicas: {{ .Values.audit.replicaCount }}
+ selector:
+ matchLabels:
+ {{- include "inlong.matchLabels" . | nindent 6 }}
+ component: {{ .Values.audit.component }}
+ template:
+ metadata:
+ labels:
+ {{- include "inlong.template.labels" . | nindent 8 }}
+ component: {{ .Values.audit.component }}
+ spec:
+ initContainers:
+ - name: wait-{{ .Values.mysql.component }}-ready
+ image: {{ .Values.images.initContainer.repository }}:{{ .Values.images.initContainer.tag }}
+ imagePullPolicy: {{ .Values.images.pullPolicy }}
+ command: [ "/bin/sh", "-c" ]
+ args:
+ - |
+ count={{ .Values.mysql.replicaCount }}
+ for i in $(seq 0 $(expr $count - 1))
+ do
+ replica="{{ template "inlong.fullname" . }}-{{ .Values.mysql.component }}-$i"
+ host="$replica.{{ template "inlong.fullname" . }}-{{ .Values.mysql.component }}.{{ .Release.Namespace }}.svc.cluster.local"
+ port={{ .Values.mysql.ports.server }}
+ until [ $(nc -z -w 5 $host $port; echo $?) -eq 0 ]
+ do
+ echo "waiting for $replica to be ready"
+ sleep 3
+ done
+ done
+ - name: wait-{{ .Values.manager.component }}-ready
+ image: {{ .Values.images.initContainer.repository }}:{{ .Values.images.initContainer.tag }}
+ imagePullPolicy: {{ .Values.images.pullPolicy }}
+ command: [ "/bin/sh", "-c" ]
+ args:
+ - |
+ count={{ .Values.manager.replicaCount }}
+ for i in $(seq 0 $(expr $count - 1))
+ do
+ replica="{{ template "inlong.fullname" . }}-{{ .Values.manager.component }}-$i"
+ host="$replica.{{ template "inlong.fullname" . }}-{{ .Values.manager.component }}.{{ .Release.Namespace }}.svc.cluster.local"
+ port={{ .Values.manager.ports.webNodePort }}
+ until [ $(nc -z -w 5 $host $port; echo $?) -eq 0 ]
+ do
+ echo "waiting for $replica to be ready"
+ sleep 3
+ done
+ done
+ containers:
+ - name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ image: {{ .Values.images.audit.repository }}:{{ .Values.images.audit.tag }}
+ imagePullPolicy: {{ .Values.images.pullPolicy }}
+ env:
+ - name: JDBC_URL
+ value: "jdbc:mysql://{{ include "inlong.mysql.hostname" . }}:{{ .Values.mysql.ports.server }}/apache_inlong_audit?useSSL=false&allowPublicKeyRetrieval=true&characterEncoding=UTF-8&nullCatalogMeansCurrent=true&serverTimezone=GMT%2b8"
+ - name: USERNAME
+ valueFrom:
+ configMapKeyRef:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ key: mysqlUsername
+ - name: PASSWORD
+ {{- if eq .Values.mysql.username "root" }}
+ valueFrom:
+ secretKeyRef:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.mysql.component }}
+ key: mysql-root-password
+ {{- else }}
+ value: {{ .Values.mysql.password | quote }}
+ {{- end }}
+ - name: MANAGER_OPENAPI_IP
+ value: {{ include "inlong.tubemqManager.hostname" . | quote }}
+ - name: MANAGER_OPENAPI_PORT
+ valueFrom:
+ configMapKeyRef:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ key: managerOpenApiPort
+ - name: PULSAR_BROKER_URL_LIST
+ valueFrom:
+ configMapKeyRef:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.audit.component }}
+ key: pulsarBrokerUrlList
+ {{- range $key, $value := .Values.audit.env }}
+ - name: {{ $key }}
+ value: {{ $value | quote }}
+ {{- end }}
+ ports:
+ - name: {{ .Values.audit.component }}-port
+ containerPort: 8008
+ volumeMounts:
+ - name: data
+ mountPath: /data/collect-data
+ volumes:
+ - name: data
+ emptyDir: {}
+ restartPolicy: Always
diff --git a/docker/kubernetes/templates/pulsar-pvc.yaml b/docker/kubernetes/templates/pulsar-pvc.yaml
new file mode 100644
index 0000000..aed4ad0
--- /dev/null
+++ b/docker/kubernetes/templates/pulsar-pvc.yaml
@@ -0,0 +1,36 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+{{- if .Values.pulsar.persistentVolumeClaim.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "inlong.commonLabels" . | nindent 4 }}
+ component: {{ .Values.pulsar.component }}
+spec:
+ accessModes:
+ {{- range .Values.pulsar.persistentVolumeClaim.accessModes }}
+ - {{ . | quote }}
+ {{- end }}
+ resources:
+ requests:
+ storageClassName: {{ .Values.pulsar.persistentVolumeClaim.storageClassName | quote }}
+ storage: {{ .Values.pulsar.persistentVolumeClaim.storage | quote }}
+{{- end }}
diff --git a/docker/kubernetes/templates/pulsar-service.yaml b/docker/kubernetes/templates/pulsar-service.yaml
new file mode 100644
index 0000000..6e5dc62
--- /dev/null
+++ b/docker/kubernetes/templates/pulsar-service.yaml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: Service
+metadata:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "inlong.commonLabels" . | nindent 4 }}
+ component: {{ .Values.pulsar.component }}
+spec:
+ ports:
+ - name: {{ .Values.pulsar.component }}-port
+ port: {{ .Values.pulsar.ports.pulsarPort }}
+ clusterIP: None
+ selector:
+ {{- include "inlong.matchLabels" . | nindent 4 }}
+ component: {{ .Values.pulsar.component }}
diff --git a/docker/kubernetes/templates/pulsar-statefulset.yaml b/docker/kubernetes/templates/pulsar-statefulset.yaml
new file mode 100644
index 0000000..bb5b39b
--- /dev/null
+++ b/docker/kubernetes/templates/pulsar-statefulset.yaml
@@ -0,0 +1,66 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ namespace: {{ .Release.Namespace }}
+ labels:
+ {{- include "inlong.commonLabels" . | nindent 4 }}
+ component: {{ .Values.pulsar.component }}
+spec:
+ serviceName: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ replicas: {{ .Values.pulsar.replicaCount }}
+ selector:
+ matchLabels:
+ {{- include "inlong.matchLabels" . | nindent 6 }}
+ component: {{ .Values.pulsar.component }}
+ template:
+ metadata:
+ labels:
+ {{- include "inlong.template.labels" . | nindent 8 }}
+ component: {{ .Values.pulsar.component }}
+ spec:
+ containers:
+ - name: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ image: {{ .Values.images.pulsar.repository }}:{{ .Values.images.pulsar.tag }}
+ imagePullPolicy: {{ .Values.images.pullPolicy }}
+ ports:
+ - name: {{ .Values.pulsar.component }}-port
+ containerPort: 6650
+ volumeMounts:
+ - name: data
+ mountPath: /pulsar/data
+ - name: conf
+ mountPath: /pulsar/conf
+ volumes:
+ - name: data
+ {{- if .Values.pulsar.persistentVolumeClaim.enabled }}
+ persistentVolumeClaim:
+ claimName: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ {{- else }}
+ emptyDir: {}
+ {{- end }}
+ - name: conf
+ {{- if .Values.pulsar.persistentVolumeClaim.enabled }}
+ persistentVolumeClaim:
+ claimName: {{ template "inlong.fullname" . }}-{{ .Values.pulsar.component }}
+ {{- else }}
+ emptyDir: {}
+ {{- end }}
+ restartPolicy: Always
diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml
index ee5da37..02144de 100644
--- a/docker/kubernetes/values.yaml
+++ b/docker/kubernetes/values.yaml
@@ -36,6 +36,12 @@ images:
manager:
repository: inlong/manager
tag: latest
+ audit:
+ repository: inlong/audit
+ tag: latest
+ pulsar:
+ repository: apachepulsar/pulsar
+ tag: latest
mysql:
repository: mysql
tag: 5.7
@@ -101,9 +107,33 @@ manager:
service:
type: LoadBalancer
+audit:
+ component: "audit"
+ replicaCount: 1
+ ports:
+ webPort: 8008
+ env:
+
+pulsar:
+ component: "pulsar"
+ replicaCount: 1
+ ports:
+ pulsarPort: 6650
+ persistentVolumeClaim:
+ enabled: false
+ accessModes:
+ - "ReadWriteOnce"
+ # storageClassName must support the above access mode(s): ReadWriteOnce
+ storageClassName: "-"
+ storage: "8Gi"
+
mysql:
component: "mysql"
replicaCount: 1
+ username: "root"
+ # If the MySQL username is root,
+ # the password parameter can be omitted and the rootPassword will be used.
+ password: "inlong"
rootPassword: "inlong"
ports:
server: 3306