You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ke...@apache.org on 2022/11/11 02:27:02 UTC
[dolphinscheduler] branch dev updated: Add minio as demo storage for helm charts (#12861)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 454f1303a0 Add minio as demo storage for helm charts (#12861)
454f1303a0 is described below
commit 454f1303a099736663eed62e327229a05573aacf
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Fri Nov 11 10:26:57 2022 +0800
Add minio as demo storage for helm charts (#12861)
---
deploy/kubernetes/dolphinscheduler/Chart.yaml | 4 +++
.../resources/config/common.properties | 21 -------------
.../dolphinscheduler/templates/NOTES.txt | 34 +++++++++++-----------
.../dolphinscheduler/templates/_helpers.tpl | 21 +++++++++++++
.../dolphinscheduler/templates/configmap.yaml | 10 ++++++-
deploy/kubernetes/dolphinscheduler/values.yaml | 15 ++++++++--
docs/docs/en/guide/installation/kubernetes.md | 1 +
7 files changed, 64 insertions(+), 42 deletions(-)
diff --git a/deploy/kubernetes/dolphinscheduler/Chart.yaml b/deploy/kubernetes/dolphinscheduler/Chart.yaml
index 94e5b312ec..2f93e51d81 100644
--- a/deploy/kubernetes/dolphinscheduler/Chart.yaml
+++ b/deploy/kubernetes/dolphinscheduler/Chart.yaml
@@ -60,3 +60,7 @@ dependencies:
version: 9.4.1
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
condition: mysql.enabled
+- name: minio
+ version: 2022.10.29
+ repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+ condition: minio.enabled
diff --git a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties b/deploy/kubernetes/dolphinscheduler/resources/config/common.properties
deleted file mode 100644
index a152640c0f..0000000000
--- a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if index .Values.conf "common" }}
-{{- range $key, $value := index .Values.conf "common" }}
-{{ $key }}={{ $value }}
-{{- end }}
-{{- end }}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
index ac15c07b53..ab42cf2e12 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
+++ b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
@@ -1,19 +1,19 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/}}
** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed **
@@ -47,4 +47,4 @@ Access DolphinScheduler UI URL by:
DolphinScheduler UI URL: http://$SERVICE_IP:12345/dolphinscheduler
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
index 0f36875d56..dd23b8d184 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
+++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
@@ -117,6 +117,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
+{{/*
+Create a default fully qualified minio name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "dolphinscheduler.minio.fullname" -}}
+{{- $name := default "minio" .Values.minio.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
{{/*
Create a default fully qualified zookkeeper quorum.
*/}}
@@ -184,6 +193,18 @@ Wait for database to be ready.
{{- end }}
{{- end -}}
+{{/*
+Wait for minio to be ready.
+*/}}
+{{- define "dolphinscheduler.minio.wait-for-ready" -}}
+{{- if .Values.minio.enabled }}
+- name: wait-for-minio
+ image: busybox:1.30
+ imagePullPolicy: IfNotPresent
+ command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ template "dolphinscheduler.minio.fullname" . }} 9000 && exit 0 || sleep 5; done; exit 1']
+{{- end }}
+{{- end -}}
+
{{/*
Create a registry environment variables.
*/}}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
index 57d5155c37..52c2730341 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
+++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
@@ -23,4 +23,12 @@ metadata:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
data:
common_properties: |-
-{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }}
\ No newline at end of file
+ {{- if index .Values.conf "common" }}
+ {{- range $key, $value := index .Values.conf "common" }}
+ {{- if and $.Values.minio.enabled }}
+ {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
+ {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
+ {{- end }}
+ {{ $key }}={{ $value }}
+ {{- end }}
+ {{- end }}
diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml
index 46b015644d..555ceefef3 100644
--- a/deploy/kubernetes/dolphinscheduler/values.yaml
+++ b/deploy/kubernetes/dolphinscheduler/values.yaml
@@ -57,6 +57,15 @@ mysql:
size: "20Gi"
storageClass: "-"
+minio:
+ enabled: false
+ auth:
+ rootUser: minioadmin
+ rootPassword: minioadmin
+ persistence:
+ enabled: false
+ defaultBuckets: "dolphinscheduler"
+
## If exists external database, and set postgresql.enable value to false.
## external database will be used, otherwise Dolphinscheduler's database will be used.
externalDatabase:
@@ -91,7 +100,7 @@ conf:
data.basedir.path: /tmp/dolphinscheduler
# resource storage type: HDFS, S3, NONE
- resource.storage.type: HDFS
+ resource.storage.type: NONE
# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path: /dolphinscheduler
@@ -103,13 +112,13 @@ conf:
resource.aws.secret.access.key: minioadmin
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
- resource.aws.region: cn-north-1
+ resource.aws.region: ca-central-1
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name: dolphinscheduler
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
- resource.aws.s3.endpoint: http://localhost:9000
+ resource.aws.s3.endpoint: http://minio:9000
# alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id: <your-access-key-id>
diff --git a/docs/docs/en/guide/installation/kubernetes.md b/docs/docs/en/guide/installation/kubernetes.md
index 8671e65a4e..cbe2233f6d 100644
--- a/docs/docs/en/guide/installation/kubernetes.md
+++ b/docs/docs/en/guide/installation/kubernetes.md
@@ -495,6 +495,7 @@ common:
| `postgresql.persistence.enabled` | Set `postgresql.persistence.enabled` to `true` to mount a new volume for internal PostgreSQL | `false` |
| `postgresql.persistence.size` | `PersistentVolumeClaim` size | `20Gi` |
| `postgresql.persistence.storageClass` | PostgreSQL data persistent volume storage class. If set to "-", storageClassName: "", which disables dynamic provisioning | `-` |
+| `minio.enabled` | Deploy minio and configure it as the default storage for DolphinScheduler, note this is for demo only, not for production. | `false` |
| `externalDatabase.type` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database type will use it | `postgresql` |
| `externalDatabase.driver` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database driver will use it | `org.postgresql.Driver` |
| `externalDatabase.host` | If exists external PostgreSQL, and set `postgresql.enabled` value to false. DolphinScheduler's database host will use it | `localhost` |