You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sdap.apache.org by ea...@apache.org on 2020/07/27 17:29:44 UTC
[incubator-sdap-nexus] branch bug_fixes updated: cassandra helm
chart included
This is an automated email from the ASF dual-hosted git repository.
eamonford pushed a commit to branch bug_fixes
in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
The following commit(s) were added to refs/heads/bug_fixes by this push:
new 94ad241 cassandra helm chart included
94ad241 is described below
commit 94ad24110420d524c98aa11db17089370f62f7c4
Author: Eamon Ford <ea...@jpl.nasa.gov>
AuthorDate: Mon Jul 27 10:29:33 2020 -0700
cassandra helm chart included
---
data-access/nexustiles/dao/CassandraProxy.py | 8 +-
helm/requirements.yaml | 5 +-
helm/templates/cassandra.yml | 107 ---------------------------
helm/templates/granule-ingester.yml | 4 +
helm/templates/init-cassandra-configmap.yml | 13 ++++
helm/templates/webapp.yml | 2 +-
helm/values.yaml | 30 +++++---
7 files changed, 45 insertions(+), 124 deletions(-)
diff --git a/data-access/nexustiles/dao/CassandraProxy.py b/data-access/nexustiles/dao/CassandraProxy.py
index 2b27104..2189efd 100644
--- a/data-access/nexustiles/dao/CassandraProxy.py
+++ b/data-access/nexustiles/dao/CassandraProxy.py
@@ -13,20 +13,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+import logging
import uuid
from ConfigParser import NoOptionError
-from multiprocessing.synchronize import Lock
-from cassandra.auth import PlainTextAuthProvider
import nexusproto.DataTile_pb2 as nexusproto
import numpy as np
+from cassandra.auth import PlainTextAuthProvider
from cassandra.cqlengine import columns, connection, CQLEngineException
from cassandra.cqlengine.models import Model
from cassandra.policies import TokenAwarePolicy, DCAwareRoundRobinPolicy, WhiteListRoundRobinPolicy
+from multiprocessing.synchronize import Lock
from nexusproto.serialization import from_shaped_array
INIT_LOCK = Lock()
+logger = logging.getLogger(__name__)
class NexusTileData(Model):
__table_name__ = 'sea_surface_temp'
@@ -171,7 +173,7 @@ class CassandraProxy(object):
self.__open()
def __open(self):
-
+ logger.info("Connecting to cassandra at " + self.__cass_url)
if self.__cass_dc_policy == 'DCAwareRoundRobinPolicy':
dc_policy = DCAwareRoundRobinPolicy(self.__cass_local_DC)
elif self.__cass_dc_policy == 'WhiteListRoundRobinPolicy':
diff --git a/helm/requirements.yaml b/helm/requirements.yaml
index ebcd9c0..3e038e7 100644
--- a/helm/requirements.yaml
+++ b/helm/requirements.yaml
@@ -10,6 +10,7 @@ dependencies:
- name: solr
version: 1.5.2
repository: http://storage.googleapis.com/kubernetes-charts-incubator
-
-
+ - name: cassandra
+ version: 5.5.3
+ repository: https://charts.bitnami.com/bitnami
diff --git a/helm/templates/cassandra.yml b/helm/templates/cassandra.yml
deleted file mode 100644
index 6023e55..0000000
--- a/helm/templates/cassandra.yml
+++ /dev/null
@@ -1,107 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: sdap-cassandra
-spec:
- clusterIP: None
- ports:
- - name: cql
- port: 9042
- targetPort: cql
- selector:
- app: sdap-cassandra
-
----
-
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: cassandra-set
-spec:
- serviceName: sdap-cassandra
- replicas: {{ .Values.cassandra.replicas }}
- selector:
- matchLabels:
- app: sdap-cassandra
- template:
- metadata:
- labels:
- app: sdap-cassandra
- spec:
- terminationGracePeriodSeconds: 120
- {{ if .Values.cassandra.tolerations }}
- tolerations:
-{{ .Values.cassandra.tolerations | toYaml | indent 6 }}
- {{ end }}
- {{ if .Values.cassandra.nodeSelector }}
- nodeSelector:
-{{ .Values.cassandra.nodeSelector | toYaml | indent 8 }}
- {{ end }}
- affinity:
- podAntiAffinity:
- # Prefer spreading over all hosts
- preferredDuringSchedulingIgnoredDuringExecution:
- - weight: 100
- podAffinityTerm:
- labelSelector:
- matchExpressions:
- - key: "app"
- operator: In
- values:
- - sdap-cassandra
- topologyKey: "kubernetes.io/hostname"
- containers:
- - name: cassandra
- image: nexusjpl/cassandra:1.0.0-rc1
- imagePullPolicy: Always
- ports:
- - containerPort: 7000
- name: intra-node
- - containerPort: 7001
- name: tls-intra-node
- - containerPort: 7199
- name: jmx
- - containerPort: 9042
- name: cql
- resources:
- requests:
- cpu: {{ .Values.cassandra.requests.cpu }}
- memory: {{ .Values.cassandra.requests.memory }}
- limits:
- cpu: {{ .Values.cassandra.limits.cpu }}
- memory: {{ .Values.cassandra.limits.memory }}
- securityContext:
- capabilities:
- add:
- - IPC_LOCK
- lifecycle:
- preStop:
- exec:
- command:
- - /bin/sh
- - -c
- - nodetool drain
- env:
- - name: MAX_HEAP_SIZE
- value: 2G
- - name: HEAP_NEWSIZE
- value: 200M
- - name: CASSANDRA_SEEDS
- value: "cassandra-set-0.sdap-cassandra"
- - name: POD_IP
- valueFrom:
- fieldRef:
- fieldPath: status.podIP
- volumeMounts:
- - name: cassandra-data
- mountPath: /var/lib/cassandra
-
- volumeClaimTemplates:
- - metadata:
- name: cassandra-data
- spec:
- accessModes: [ "ReadWriteOnce" ]
- storageClassName: {{ .Values.storageClass }}
- resources:
- requests:
- storage: {{ .Values.cassandra.storage }}
diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml
index 2a716f1..312e7dd 100644
--- a/helm/templates/granule-ingester.yml
+++ b/helm/templates/granule-ingester.yml
@@ -28,6 +28,10 @@ spec:
value: {{ .Values.rabbitmq.fullnameOverride }}
- name: CASSANDRA_CONTACT_POINTS
value: sdap-cassandra
+ - name: CASSANDRA_USERNAME
+ value: cassandra
+ - name: CASSANDRA_PASSWORD
+ value: cassandra
- name: ZK_HOST_AND_PORT
value: {{ .Release.Name }}-zookeeper:2181
{{ if .Values.ingestion.granuleIngester.maxConcurrency }}
diff --git a/helm/templates/init-cassandra-configmap.yml b/helm/templates/init-cassandra-configmap.yml
new file mode 100644
index 0000000..eadbc9c
--- /dev/null
+++ b/helm/templates/init-cassandra-configmap.yml
@@ -0,0 +1,13 @@
+apiVersion: v1
+data:
+ init.cql: |
+ CREATE KEYSPACE IF NOT EXISTS nexustiles WITH REPLICATION = { 'class': 'SimpleStrategy', 'replication_factor': 1 };
+
+ CREATE TABLE IF NOT EXISTS nexustiles.sea_surface_temp (
+ tile_id uuid PRIMARY KEY,
+ tile_blob blob
+ );
+kind: ConfigMap
+metadata:
+ name: init-cassandra
+ namespace: sdap
diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml
index 2963cce..8efd3ec 100644
--- a/helm/templates/webapp.yml
+++ b/helm/templates/webapp.yml
@@ -12,7 +12,7 @@ spec:
imagePullPolicy: IfNotPresent
mainApplicationFile: local:///incubator-sdap-nexus/analysis/webservice/webapp.py
arguments:
- - "--cassandra-host=sdap-cassandra --solr-host={{ .Release.Name }}-solr-svc:8983"
+ - "--cassandra-host={{ .Release.Name }}-cassandra --cassandra-username=cassandra --cassandra-password=cassandra --solr-host={{ .Release.Name }}-solr-svc:8983"
sparkVersion: "2.4.4"
restartPolicy:
type: OnFailure
diff --git a/helm/values.yaml b/helm/values.yaml
index aa03c0a..c9b9cf1 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -99,16 +99,6 @@ ingestion:
## Store ingestion history in a solr database instead of a filesystem directory
# url: http://history-solr
-cassandra:
- replicas: 2
- storage: 13Gi
- requests:
- cpu: 1
- memory: 3Gi
- limits:
- cpu: 1
- memory: 3Gi
-
solr:
replicaCount: 3
volumeClaimTemplates:
@@ -165,4 +155,22 @@ rabbitmq:
username: guest
password: guest
ingress:
- enabled: true
\ No newline at end of file
+ enabled: true
+
+cassandra:
+ initDBConfigMap: init-cassandra
+ dbUser:
+ user: cassandra
+ password: cassandra
+ cluster:
+ replicaCount: 1
+ persistence:
+ storageClass: hostpath
+ size: 8Gi
+ resources:
+ requests:
+ cpu: 1
+ memory: 8Gi
+ limits:
+ cpu: 1
+ memory: 8Gi