You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ke...@apache.org on 2022/11/10 09:32:38 UTC
[dolphinscheduler] branch helm/minio created (now 4ae6f0ebdd)
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a change to branch helm/minio
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
at 4ae6f0ebdd Add minio as demo storage for helm charts
This branch includes the following new commits:
new 4ae6f0ebdd Add minio as demo storage for helm charts
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[dolphinscheduler] 01/01: Add minio as demo storage for helm charts
Posted by ke...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
kezhenxu94 pushed a commit to branch helm/minio
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 4ae6f0ebdd615059eca02e6b3f62dedf0569dcae
Author: kezhenxu94 <ke...@apache.org>
AuthorDate: Thu Nov 10 17:32:25 2022 +0800
Add minio as demo storage for helm charts
---
deploy/kubernetes/dolphinscheduler/Chart.yaml | 4 +++
.../resources/config/common.properties | 21 -------------
.../dolphinscheduler/templates/NOTES.txt | 34 +++++++++++-----------
.../dolphinscheduler/templates/_helpers.tpl | 21 +++++++++++++
.../dolphinscheduler/templates/configmap.yaml | 10 ++++++-
.../job-dolphinscheduler-minio-initializer.yaml | 0
deploy/kubernetes/dolphinscheduler/values.yaml | 15 ++++++++--
7 files changed, 63 insertions(+), 42 deletions(-)
diff --git a/deploy/kubernetes/dolphinscheduler/Chart.yaml b/deploy/kubernetes/dolphinscheduler/Chart.yaml
index 94e5b312ec..2f93e51d81 100644
--- a/deploy/kubernetes/dolphinscheduler/Chart.yaml
+++ b/deploy/kubernetes/dolphinscheduler/Chart.yaml
@@ -60,3 +60,7 @@ dependencies:
version: 9.4.1
repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
condition: mysql.enabled
+- name: minio
+ version: 2022.10.29
+ repository: https://raw.githubusercontent.com/bitnami/charts/archive-full-index/bitnami
+ condition: minio.enabled
diff --git a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties b/deploy/kubernetes/dolphinscheduler/resources/config/common.properties
deleted file mode 100644
index a152640c0f..0000000000
--- a/deploy/kubernetes/dolphinscheduler/resources/config/common.properties
+++ /dev/null
@@ -1,21 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-{{- if index .Values.conf "common" }}
-{{- range $key, $value := index .Values.conf "common" }}
-{{ $key }}={{ $value }}
-{{- end }}
-{{- end }}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
index ac15c07b53..ab42cf2e12 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
+++ b/deploy/kubernetes/dolphinscheduler/templates/NOTES.txt
@@ -1,19 +1,19 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
+{{/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/}}
** Please be patient while the chart DolphinScheduler {{ .Chart.AppVersion }} is being deployed **
@@ -47,4 +47,4 @@ Access DolphinScheduler UI URL by:
DolphinScheduler UI URL: http://$SERVICE_IP:12345/dolphinscheduler
-{{- end }}
\ No newline at end of file
+{{- end }}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
index 0f36875d56..dd23b8d184 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
+++ b/deploy/kubernetes/dolphinscheduler/templates/_helpers.tpl
@@ -117,6 +117,15 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
+{{/*
+Create a default fully qualified minio name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+*/}}
+{{- define "dolphinscheduler.minio.fullname" -}}
+{{- $name := default "minio" .Values.minio.nameOverride -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
{{/*
Create a default fully qualified zookkeeper quorum.
*/}}
@@ -184,6 +193,18 @@ Wait for database to be ready.
{{- end }}
{{- end -}}
+{{/*
+Wait for minio to be ready.
+*/}}
+{{- define "dolphinscheduler.minio.wait-for-ready" -}}
+{{- if .Values.minio.enabled }}
+- name: wait-for-minio
+ image: busybox:1.30
+ imagePullPolicy: IfNotPresent
+ command: ['sh', '-xc', 'for i in $(seq 1 180); do nc -z -w3 {{ template "dolphinscheduler.minio.fullname" . }} 9000 && exit 0 || sleep 5; done; exit 1']
+{{- end }}
+{{- end -}}
+
{{/*
Create a registry environment variables.
*/}}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
index 57d5155c37..52c2730341 100644
--- a/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
+++ b/deploy/kubernetes/dolphinscheduler/templates/configmap.yaml
@@ -23,4 +23,12 @@ metadata:
{{- include "dolphinscheduler.common.labels" . | nindent 4 }}
data:
common_properties: |-
-{{ tpl (.Files.Get "resources/config/common.properties") . | indent 4 }}
\ No newline at end of file
+ {{- if index .Values.conf "common" }}
+ {{- range $key, $value := index .Values.conf "common" }}
+ {{- if and $.Values.minio.enabled }}
+ {{- if eq $key "resource.storage.type" }}{{ $value = "S3" }}{{- end }}
+ {{- if eq $key "resource.aws.s3.endpoint" }}{{ $value = print "http://" (include "dolphinscheduler.minio.fullname" $) ":9000" }}{{- end }}
+ {{- end }}
+ {{ $key }}={{ $value }}
+ {{- end }}
+ {{- end }}
diff --git a/deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-minio-initializer.yaml b/deploy/kubernetes/dolphinscheduler/templates/job-dolphinscheduler-minio-initializer.yaml
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/deploy/kubernetes/dolphinscheduler/values.yaml b/deploy/kubernetes/dolphinscheduler/values.yaml
index 46b015644d..555ceefef3 100644
--- a/deploy/kubernetes/dolphinscheduler/values.yaml
+++ b/deploy/kubernetes/dolphinscheduler/values.yaml
@@ -57,6 +57,15 @@ mysql:
size: "20Gi"
storageClass: "-"
+minio:
+ enabled: false
+ auth:
+ rootUser: minioadmin
+ rootPassword: minioadmin
+ persistence:
+ enabled: false
+ defaultBuckets: "dolphinscheduler"
+
## If exists external database, and set postgresql.enable value to false.
## external database will be used, otherwise Dolphinscheduler's database will be used.
externalDatabase:
@@ -91,7 +100,7 @@ conf:
data.basedir.path: /tmp/dolphinscheduler
# resource storage type: HDFS, S3, NONE
- resource.storage.type: HDFS
+ resource.storage.type: NONE
# resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
resource.storage.upload.base.path: /dolphinscheduler
@@ -103,13 +112,13 @@ conf:
resource.aws.secret.access.key: minioadmin
# The AWS Region to use. if resource.storage.type=S3 or use EMR-Task, This configuration is required
- resource.aws.region: cn-north-1
+ resource.aws.region: ca-central-1
# The name of the bucket. You need to create them by yourself. Otherwise, the system cannot start. All buckets in Amazon S3 share a single namespace; ensure the bucket is given a unique name.
resource.aws.s3.bucket.name: dolphinscheduler
# You need to set this parameter when private cloud s3. If S3 uses public cloud, you only need to set resource.aws.region or set to the endpoint of a public cloud such as S3.cn-north-1.amazonaws.com.cn
- resource.aws.s3.endpoint: http://localhost:9000
+ resource.aws.s3.endpoint: http://minio:9000
# alibaba cloud access key id, required if you set resource.storage.type=OSS
resource.alibaba.cloud.access.key.id: <your-access-key-id>