You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/11/10 23:16:43 UTC
[incubator-sdap-nexus] branch bug_fixes updated: Add S3 support to
helm chart
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch bug_fixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/bug_fixes by this push:
new 2dc0780 Add S3 support to helm chart
2dc0780 is described below
commit 2dc0780e68424c3476dbb4d1da72188a24f5d45e
Author: Eamon Ford <ea...@gmail.com>
AuthorDate: Tue Nov 10 15:16:29 2020 -0800
Add S3 support to helm chart
---
.gitignore | 1 +
helm/Chart.yaml | 2 +-
helm/templates/collection-manager.yml | 16 +++++++++++++++-
helm/templates/granule-ingester.yml | 8 ++++++++
helm/values.yaml | 15 ++++++++++++---
5 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/.gitignore b/.gitignore
index 3e29626..12ab2d6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
+*.pytest_cache
*.vscode
*.code-workspace
*.idea
diff --git a/helm/Chart.yaml b/helm/Chart.yaml
index 2a2e9d1..6689340 100644
--- a/helm/Chart.yaml
+++ b/helm/Chart.yaml
@@ -1,5 +1,5 @@
apiVersion: v1
-appVersion: "0.1.5"
+appVersion: "0.2.1"
description: Science Data Analytics Platform
name: nexus
version: 0.2.0
diff --git a/helm/templates/collection-manager.yml b/helm/templates/collection-manager.yml
index e281526..a116dd0 100644
--- a/helm/templates/collection-manager.yml
+++ b/helm/templates/collection-manager.yml
@@ -1,5 +1,7 @@
{{- if .Values.ingestion.enabled }}
{{- $history := .Values.ingestion.history | default dict }}
+{{- $s3 := .Values.ingestion.granules.s3 | default dict }}
+{{ $aws_creds_envs := $s3.awsCredsEnvs | default dict }}
apiVersion: apps/v1
kind: Deployment
@@ -37,6 +39,14 @@ spec:
- name: HISTORY_PATH
value: {{ include "nexus.history.mountPath" . }}
{{- end }}
+ {{- if $s3.bucket }}
+ - name: S3_BUCKET
+ value: {{ $s3.bucket }}
+ {{- end }}
+ {{- range $name, $value := $aws_creds_envs }}
+ - name: {{ $name }}
+ value: {{ $value }}
+ {{- end }}
resources:
requests:
cpu: {{ .Values.ingestion.collectionManager.cpu }}
@@ -45,15 +55,19 @@ spec:
cpu: {{ .Values.ingestion.collectionManager.cpu }}
memory: {{ .Values.ingestion.collectionManager.memory }}
volumeMounts:
-{{ include "nexus.ingestion.dataVolumeMount" . | indent 12 }}
{{- if not $history.solrEnabled }}
- name: history-volume
mountPath: {{ include "nexus.history.mountPath" . }}
{{- end }}
- name: collections-config-volume
mountPath: {{ include "nexus.collectionsConfig.mountPath" . }}
+{{- if not $s3.bucket }}
+{{ include "nexus.ingestion.dataVolumeMount" . | indent 12 }}
+{{- end }}
volumes:
+{{- if not $s3.bucket }}
{{ include "nexus.ingestion.dataVolume" . | indent 8 }}
+{{- end }}
- name: collections-config-volume
configMap:
name: {{ include "nexus.collectionsConfig.configmapName" . }}
diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml
index bb616ad..323a5fb 100644
--- a/helm/templates/granule-ingester.yml
+++ b/helm/templates/granule-ingester.yml
@@ -1,4 +1,8 @@
{{- if .Values.ingestion.enabled }}
+
+{{- $s3 := .Values.ingestion.granules.s3 | default dict }}
+{{ $aws_creds_envs := $s3.awsCredsEnvs | default dict }}
+
apiVersion: apps/v1
kind: Deployment
metadata:
@@ -38,6 +42,10 @@ spec:
- name: MAX_CONCURRENCY
value: "{{ .Values.ingestion.granuleIngester.maxConcurrency }}"
{{ end }}
+ {{- range $name, $value := $aws_creds_envs }}
+ - name: {{ $name }}
+ value: {{ $value }}
+ {{- end }}
resources:
requests:
cpu: {{ .Values.ingestion.granuleIngester.cpu }}
diff --git a/helm/values.yaml b/helm/values.yaml
index 041b0ba..32df703 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -8,7 +8,7 @@ rootWebpage:
webapp:
enabled: true
distributed:
- image: nexusjpl/nexus-webapp:distributed.0.2.0
+ image: nexusjpl/nexus-webapp:distributed.0.2.1
## Use any of the driver configuration options available at
## https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md
@@ -31,7 +31,7 @@ ingestion:
granuleIngester:
replicas: 2
- image: nexusjpl/granule-ingester:0.0.4
+ image: nexusjpl/granule-ingester:0.1.0a3
## cpu refers to both request and limit
cpu: 1
@@ -40,7 +40,7 @@ ingestion:
memory: 1Gi
collectionManager:
- image: nexusjpl/collection-manager:0.0.4
+ image: nexusjpl/collection-manager:0.1.0a7
## cpu refers to both request and limit
cpu: 0.5
@@ -71,6 +71,15 @@ ingestion:
## the granule files are stored.
path: /var/lib/sdap/granules
+ # s3:
+ # bucket: nexus-ingest
+ # awsCredsEnvs:
+ # AWS_ACCESS_KEY_ID: secret
+ # AWS_SECRET_ACCESS_KEY: secret
+ # AWS_SESSION_TOKEN: secret
+ # AWS_DEFAULT_REGION: us-west-2
+
+
## Where to find the Collections Config file
## ref: https://github.com/apache/incubator-sdap-ingester/tree/dev/collection_manager#the-collections-configuration-file
## Either localDir should be set, or the git options, but not both.