You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/07/21 00:03:34 UTC
[incubator-sdap-nexus] branch bug_fixes updated: use solr and zk
helm charts
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch bug_fixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/bug_fixes by this push:
new b78c8ec use solr and zk helm charts
b78c8ec is described below
commit b78c8ecb0712401ad8e7487f5db66c8543c79aed
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Mon Jul 20 17:03:23 2020 -0700
use solr and zk helm charts
---
docker/solr/cloud-init/create-collection.py | 4 +-
helm/requirements.yaml | 5 +
helm/templates/granule-ingester.yml | 3 +-
helm/templates/solr-create-collection.yml | 34 +++++++
helm/templates/solr.yml | 129 -------------------------
helm/templates/zookeeper.yml | 144 ----------------------------
helm/values.yaml | 36 ++++---
7 files changed, 65 insertions(+), 290 deletions(-)
diff --git a/docker/solr/cloud-init/create-collection.py b/docker/solr/cloud-init/create-collection.py
index f8f98bc..9e46260 100755
--- a/docker/solr/cloud-init/create-collection.py
+++ b/docker/solr/cloud-init/create-collection.py
@@ -107,5 +107,5 @@ finally:
# We're done, do nothing forever.
logging.info("Done.")
-while True:
- time.sleep(987654321)
+# while True:
+# time.sleep(987654321)
diff --git a/helm/requirements.yaml b/helm/requirements.yaml
index 7970f29..57a7e13 100644
--- a/helm/requirements.yaml
+++ b/helm/requirements.yaml
@@ -7,5 +7,10 @@ dependencies:
version: 7.1.0
repository: https://charts.bitnami.com/bitnami
condition: ingestion.enabled
+ - name: solr
+ version: 1.5.2
+ repository: http://storage.googleapis.com/kubernetes-charts-incubator
+ condition: ingestion.enabled
+
diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml
index ba4794a..078e744 100644
--- a/helm/templates/granule-ingester.yml
+++ b/helm/templates/granule-ingester.yml
@@ -17,6 +17,7 @@ spec:
spec:
containers:
- image: {{ .Values.ingestion.granuleIngester.image }}
+ imagePullPolicy: IfNotPresent
name: granule-ingester
env:
- name: RABBITMQ_USERNAME
@@ -28,7 +29,7 @@ spec:
- name: CASSANDRA_CONTACT_POINTS
value: sdap-cassandra
- name: ZK_HOST_AND_PORT
- value: zk-cs:2181
+ value: {{ .Release.Namespace }}-zookeeper:2181
resources:
requests:
cpu: {{ .Values.ingestion.granuleIngester.cpu }}
diff --git a/helm/templates/solr-create-collection.yml b/helm/templates/solr-create-collection.yml
new file mode 100644
index 0000000..6886cdd
--- /dev/null
+++ b/helm/templates/solr-create-collection.yml
@@ -0,0 +1,34 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: solr-create-collection
+spec:
+# selector:
+# matchLabels:
+# app: solr-create-collection # has to match .spec.template.metadata.labels
+# replicas: 1
+ template:
+ metadata:
+ labels:
+ app: solr-create-collection
+ spec:
+ containers:
+ - name: solr-create-collection
+ imagePullPolicy: Always
+ image: nexusjpl/solr-cloud-init:1.0.1
+ resources:
+ requests:
+ memory: "1Gi"
+ cpu: "0.25"
+ env:
+ - name: MINIMUM_NODES
+ value: "{{ .Values.solr.replicaCount }}"
+ - name: SOLR_HOST
+ value: "{{ .Release.Namespace }}-solr-svc"
+ - name: SDAP_SOLR_URL
+ value: "http://$(SOLR_HOST):8983/solr/"
+ - name: SDAP_ZK_SOLR
+ value: "{{ .Release.Namespace }}-zookeeper:2181/solr"
+ - name: CREATE_COLLECTION_PARAMS
+ value: "name=nexustiles&numShards=$(MINIMUM_NODES)&waitForFinalState=true"
+ restartPolicy: OnFailure
diff --git a/helm/templates/solr.yml b/helm/templates/solr.yml
deleted file mode 100644
index c8d0f9b..0000000
--- a/helm/templates/solr.yml
+++ /dev/null
@@ -1,129 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: sdap-solr
-spec:
- ports:
- - port: 8983
- clusterIP: None
- selector:
- app: sdap-solr
-
----
-
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: solr-set
-spec:
- selector:
- matchLabels:
- app: sdap-solr # has to match .spec.template.metadata.labels
- serviceName: "sdap-solr"
- replicas: {{.Values.solr.replicas }} # by default is 1
- podManagementPolicy: Parallel
- template:
- metadata:
- labels:
- app: sdap-solr # has to match .spec.selector.matchLabels
- spec:
- terminationGracePeriodSeconds: 10
- {{ if .Values.solr.tolerations }}
- tolerations:
-{{ .Values.solr.tolerations | toYaml | indent 6 }}
- {{ end }}
- {{ if .Values.solr.nodeSelector }}
- nodeSelector:
-{{ .Values.solr.nodeSelector | toYaml | indent 8 }}
- {{ end }}
- affinity:
- podAntiAffinity:
- # Prefer spreading over all hosts
- preferredDuringSchedulingIgnoredDuringExecution:
- - weight: 100
- podAffinityTerm:
- labelSelector:
- matchExpressions:
- - key: "app"
- operator: In
- values:
- - sdap-solr
- topologyKey: "kubernetes.io/hostname"
- securityContext:
- runAsUser: 8983
- fsGroup: 8983
- containers:
- - name: solr-create-collection
- imagePullPolicy: Always
- image: nexusjpl/solr-cloud-init:1.0.0-rc1
- resources:
- requests:
- memory: "1Gi"
- cpu: "0.25"
- env:
- - name: MINIMUM_NODES
- value: "2" # MINIMUM_NODES should be the same as spec.replicas
- - name: SOLR_HOST
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- - name: SDAP_SOLR_URL
- value: http://$(SOLR_HOST):8983/solr/
- - name: SDAP_ZK_SOLR
- value: "zk-hs:2181/solr"
- - name: CREATE_COLLECTION_PARAMS
- value: "name=nexustiles&collection.configName=nexustiles&numShards=$(MINIMUM_NODES)&waitForFinalState=true"
- - name: solr-cloud
- imagePullPolicy: Always
- image: nexusjpl/solr-cloud:1.0.0-rc1
- resources:
- requests:
- memory: {{ .Values.solr.requests.memory }}
- cpu: {{ .Values.solr.requests.cpu }}
- limits:
- memory: {{ .Values.solr.limits.memory }}
- cpu: {{ .Values.solr.limits.cpu }}
- env:
- - name: SOLR_HEAP
- value: {{ .Values.solr.heap }}
- - name: SOLR_HOST
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- - name: SDAP_ZK_SERVICE_HOST
- value: "zk-hs"
- ports:
- - containerPort: 8983
- name: http
- volumeMounts:
- - name: solr-data
- mountPath: /opt/solr/server/solr/
- readinessProbe:
- exec:
- command:
- - solr
- - healthcheck
- - -c
- - nexustiles
- - -z
- - zk-hs:2181/solr
- initialDelaySeconds: 10
- timeoutSeconds: 5
- livenessProbe:
- exec:
- command:
- - solr
- - assert
- - -s
- - http://localhost:8983/solr/
- initialDelaySeconds: 10
- timeoutSeconds: 5
- volumeClaimTemplates:
- - metadata:
- name: solr-data
- spec:
- accessModes: [ "ReadWriteOnce" ]
- storageClassName: {{ .Values.storageClass }}
- resources:
- requests:
- storage: {{ .Values.solr.storage }}
diff --git a/helm/templates/zookeeper.yml b/helm/templates/zookeeper.yml
deleted file mode 100644
index bdc3925..0000000
--- a/helm/templates/zookeeper.yml
+++ /dev/null
@@ -1,144 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: zk-hs
- labels:
- app: zk
-spec:
- ports:
- - port: 2888
- name: server
- - port: 3888
- name: leader-election
- clusterIP: None
- selector:
- app: zk
----
-apiVersion: v1
-kind: Service
-metadata:
- name: zk-cs
- labels:
- app: zk
-spec:
- ports:
- - port: 2181
- name: client
- selector:
- app: zk
----
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
- name: zk-pdb
-spec:
- selector:
- matchLabels:
- app: zk
- maxUnavailable: 1
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: zk
-spec:
- selector:
- matchLabels:
- app: zk
- serviceName: zk-hs
- replicas: {{ .Values.zookeeper.replicas }}
- updateStrategy:
- type: RollingUpdate
- podManagementPolicy: Parallel
- template:
- metadata:
- labels:
- app: zk
- spec:
- {{ if .Values.zookeeper.tolerations }}
- tolerations:
-{{ .Values.zookeeper.tolerations | toYaml | indent 6 }}
- {{ end }}
- {{ if .Values.zookeeper.nodeSelector }}
- nodeSelector:
-{{ .Values.zookeeper.nodeSelector | toYaml | indent 8 }}
- {{ end }}
- affinity:
- podAntiAffinity:
- preferredDuringSchedulingIgnoredDuringExecution:
- - weight: 100
- podAffinityTerm:
- labelSelector:
- matchExpressions:
- - key: "app"
- operator: In
- values:
- - zk
- topologyKey: "kubernetes.io/hostname"
- containers:
- - name: kubernetes-zookeeper
- imagePullPolicy: Always
- image: "k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10"
- resources:
- requests:
- memory: {{ .Values.zookeeper.memory }}
- cpu: {{ .Values.zookeeper.cpu }}
- ports:
- - containerPort: 2181
- name: client
- - containerPort: 2888
- name: server
- - containerPort: 3888
- name: leader-election
- command:
- - sh
- - -c
- - "start-zookeeper \
- --servers={{ .Values.zookeeper.replicas }} \
- --data_dir=/var/lib/zookeeper/data \
- --data_log_dir=/var/lib/zookeeper/data/log \
- --conf_dir=/opt/zookeeper/conf \
- --client_port=2181 \
- --election_port=3888 \
- --server_port=2888 \
- --tick_time=2000 \
- --init_limit=10 \
- --sync_limit=5 \
- --heap=512M \
- --max_client_cnxns=60 \
- --snap_retain_count=3 \
- --purge_interval=12 \
- --max_session_timeout=40000 \
- --min_session_timeout=4000 \
- --log_level=INFO"
- readinessProbe:
- exec:
- command:
- - sh
- - -c
- - "zookeeper-ready 2181"
- initialDelaySeconds: 10
- timeoutSeconds: 5
- livenessProbe:
- exec:
- command:
- - sh
- - -c
- - "zookeeper-ready 2181"
- initialDelaySeconds: 10
- timeoutSeconds: 5
- volumeMounts:
- - name: zkdatadir
- mountPath: /var/lib/zookeeper
- securityContext:
- runAsUser: 1000
- fsGroup: 1000
- volumeClaimTemplates:
- - metadata:
- name: zkdatadir
- spec:
- accessModes: [ "ReadWriteOnce" ]
- storageClassName: {{ .Values.storageClass }}
- resources:
- requests:
- storage: {{ .Values.zookeeper.storage }}
diff --git a/helm/values.yaml b/helm/values.yaml
index c012e6e..5ad3b5b 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -110,21 +110,29 @@ cassandra:
memory: 3Gi
solr:
- replicas: 2
+ replicaCount: 3
storage: 10Gi
- heap: 4g
- requests:
- memory: 5Gi
- cpu: 1
- limits:
- memory: 5Gi
- cpu: 1
-
-zookeeper:
- replicas: 3
- memory: 1Gi
- cpu: 0.5
- storage: 8Gi
+ volumeClaimTemplates:
+ storageClassName: hostpath
+ storageSize: 10Gi
+ resources:
+ requests:
+ memory: 2Gi
+ cpu: 1
+ limits:
+ memory: 2Gi
+ cpu: 1
+ zookeeper:
+ replicaCount: 3
+ persistence:
+ storageClass: hostpath
+ resources:
+ limits:
+ memory: 1Gi
+ cpu: 0.5
+ requests:
+ memory: 1Gi
+ cpu: 0.5
ingressEnabled: false