You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by li...@apache.org on 2021/02/06 04:02:51 UTC
[incubator-dolphinscheduler] branch 1.3.5-prepare updated:
[1.3.5-prepare][Improvement][K8s] Support resource storage when
fs.defaultFS=file:/// (#4700)
This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch 1.3.5-prepare
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.5-prepare by this push:
new 263e94d [1.3.5-prepare][Improvement][K8s] Support resource storage when fs.defaultFS=file:/// (#4700)
263e94d is described below
commit 263e94d177da6cc48b3350cca35d858a023d697d
Author: Shiwen Cheng <ch...@gmail.com>
AuthorDate: Sat Feb 6 12:02:40 2021 +0800
[1.3.5-prepare][Improvement][K8s] Support resource storage when fs.defaultFS=file:/// (#4700)
---
docker/kubernetes/dolphinscheduler/README.md | 4 +++
.../templates/deployment-dolphinscheduler-api.yaml | 11 ++++++-
.../templates/pvc-dolphinscheduler-resource.yaml | 37 ++++++++++++++++++++++
.../statefulset-dolphinscheduler-master.yaml | 2 +-
.../statefulset-dolphinscheduler-worker.yaml | 11 ++++++-
docker/kubernetes/dolphinscheduler/values.yaml | 10 +++++-
6 files changed, 71 insertions(+), 4 deletions(-)
diff --git a/docker/kubernetes/dolphinscheduler/README.md b/docker/kubernetes/dolphinscheduler/README.md
index cf69346..c4e6fab 100644
--- a/docker/kubernetes/dolphinscheduler/README.md
+++ b/docker/kubernetes/dolphinscheduler/README.md
@@ -80,6 +80,10 @@ The following tables lists the configurable parameters of the Dolphins Scheduler
| `resource.fsS3aEndpoint` | S3 endpoint when `resource.resourceStorageType` be seted to `S3` | `s3.xxx.amazonaws.com` |
| `resource.fsS3aAccessKey` | S3 access key when `resource.resourceStorageType` be seted to `S3` | `xxxxxxx` |
| `resource.fsS3aSecretKey` | S3 secret key when `resource.resourceStorageType` be seted to `S3` | `xxxxxxx` |
+| `resource.persistentVolumeClaim.enabled` | Set `resource.persistentVolumeClaim.enabled` to `true` to mount a new resource volume for `api` and `worker` | `false` |
+| `resource.persistentVolumeClaim.accessModes` | `PersistentVolumeClaim` Access Modes, must be `ReadWriteMany` | `[ReadWriteMany]` |
+| `resource.persistentVolumeClaim.storageClassName` | Resource Persistent Volume Storage Class, must support the access mode: ReadWriteMany | `-` |
+| `resource.persistentVolumeClaim.storage` | `PersistentVolumeClaim` Size | `20Gi` |
| | | |
| `master.podManagementPolicy` | PodManagementPolicy controls how pods are created during initial scale up, when replacing pods on nodes, or when scaling down | `Parallel` |
| `master.replicas` | Replicas is the desired number of replicas of the given Template | `3` |
diff --git a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
index 0db3d55..83d9981 100644
--- a/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/deployment-dolphinscheduler-api.yaml
@@ -166,6 +166,10 @@ spec:
volumeMounts:
- mountPath: "/opt/dolphinscheduler/logs"
name: {{ include "dolphinscheduler.fullname" . }}-api
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - mountPath: {{ .Values.resource.resourceUploadPath }}
+ name: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-api
{{- if .Values.api.persistentVolumeClaim.enabled }}
@@ -173,4 +177,9 @@ spec:
claimName: {{ include "dolphinscheduler.fullname" . }}-api
{{- else }}
emptyDir: {}
- {{- end }}
\ No newline at end of file
+ {{- end }}
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - name: {{ include "dolphinscheduler.fullname" . }}-resource
+ persistentVolumeClaim:
+ claimName: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml
new file mode 100644
index 0000000..3494800
--- /dev/null
+++ b/docker/kubernetes/dolphinscheduler/templates/pvc-dolphinscheduler-resource.yaml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+{{- if .Values.resource.persistentVolumeClaim.enabled }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+ name: {{ include "dolphinscheduler.fullname" . }}-resource
+ labels:
+ app.kubernetes.io/name: {{ include "dolphinscheduler.fullname" . }}-resource
+ app.kubernetes.io/instance: {{ .Release.Name }}
+ app.kubernetes.io/managed-by: {{ .Release.Service }}
+ annotations:
+ "helm.sh/resource-policy": keep
+spec:
+ accessModes:
+ {{- range .Values.resource.persistentVolumeClaim.accessModes }}
+ - {{ . | quote }}
+ {{- end }}
+ storageClassName: {{ .Values.resource.persistentVolumeClaim.storageClassName | quote }}
+ resources:
+ requests:
+ storage: {{ .Values.resource.persistentVolumeClaim.storage | quote }}
+{{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
index 7f42205..839cef5 100644
--- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-master.yaml
@@ -202,4 +202,4 @@ spec:
resources:
requests:
storage: {{ .Values.master.persistentVolumeClaim.storage | quote }}
- {{- end }}
+ {{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
index a390a7b..4e692da 100644
--- a/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
+++ b/docker/kubernetes/dolphinscheduler/templates/statefulset-dolphinscheduler-worker.yaml
@@ -211,6 +211,10 @@ spec:
- mountPath: "/opt/dolphinscheduler/conf/env/dolphinscheduler_env.sh"
subPath: "dolphinscheduler_env.sh"
name: {{ include "dolphinscheduler.fullname" . }}-worker-configmap
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - mountPath: {{ .Values.resource.resourceUploadPath }}
+ name: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
volumes:
- name: {{ include "dolphinscheduler.fullname" . }}-worker-data
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@@ -233,6 +237,11 @@ spec:
items:
- key: dolphinscheduler_env.sh
path: dolphinscheduler_env.sh
+ {{- if .Values.resource.persistentVolumeClaim.enabled }}
+ - name: {{ include "dolphinscheduler.fullname" . }}-resource
+ persistentVolumeClaim:
+ claimName: {{ include "dolphinscheduler.fullname" . }}-resource
+ {{- end }}
{{- if .Values.worker.persistentVolumeClaim.enabled }}
volumeClaimTemplates:
{{- if .Values.worker.persistentVolumeClaim.dataPersistentVolume.enabled }}
@@ -269,4 +278,4 @@ spec:
requests:
storage: {{ .Values.worker.persistentVolumeClaim.logsPersistentVolume.storage | quote }}
{{- end }}
- {{- end }}
+ {{- end }}
\ No newline at end of file
diff --git a/docker/kubernetes/dolphinscheduler/values.yaml b/docker/kubernetes/dolphinscheduler/values.yaml
index 5206450..d154c87 100644
--- a/docker/kubernetes/dolphinscheduler/values.yaml
+++ b/docker/kubernetes/dolphinscheduler/values.yaml
@@ -65,7 +65,7 @@ zookeeper:
zookeeperRoot: /dolphinscheduler
# If exists external zookeeper, and set zookeeper.enable value to false.
-# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
+# If zookeeper.enable is false, Dolphinscheduler's zookeeper will use it.
externalZookeeper:
zookeeperQuorum: "127.0.0.1:2181"
zookeeperRoot: /dolphinscheduler
@@ -77,6 +77,14 @@ resource:
fsS3aEndpoint: "s3.xxx.amazonaws.com"
fsS3aAccessKey: "xxxxxxx"
fsS3aSecretKey: "xxxxxxx"
+ # If resourceStorageType is HDFS and fsDefaultFS is file:///, persistentVolumeClaim should be enabled for resource storage
+ persistentVolumeClaim:
+ enabled: false
+ accessModes:
+ - "ReadWriteMany"
+ # storageClassName must support the access mode: ReadWriteMany
+ storageClassName: "-"
+ storage: "20Gi"
master:
podManagementPolicy: "Parallel"