You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/05/15 13:48:51 UTC
[GitHub] csantanapr closed pull request #199: helm improvements
csantanapr closed pull request #199: helm improvements
URL: https://github.com/apache/incubator-openwhisk-deploy-kube/pull/199
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/helm/README.md b/helm/README.md
index eff997d..c774a2e 100644
--- a/helm/README.md
+++ b/helm/README.md
@@ -88,7 +88,13 @@ where `minikube ip` returns `192.168.99.100`.
whisk:
ingress:
api_host: 192.168.99.100:31001
- apigw_url: 192.168.99.100:31004
+ apigw_url: http://192.168.99.100:31004
+
+nginx:
+ httpsNodePort: 31001
+
+apigw:
+ apiNodePort: 31004
```
Deployment can be done by using the following single command:
diff --git a/helm/templates/_affinity.tpl b/helm/templates/_affinity.tpl
index 8f3267d..efa0d73 100644
--- a/helm/templates/_affinity.tpl
+++ b/helm/templates/_affinity.tpl
@@ -1,7 +1,7 @@
# This file defines template snippets for scheduler affinity and anti-affinity
-{{/* Generic control-plane affinity */}}
-{{- define "affinity.controlPlane" -}}
+{{/* Generic core affinity */}}
+{{- define "affinity.core" -}}
# prefer to not run on an invoker node (only prefer because of single node clusters)
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
@@ -12,7 +12,7 @@ nodeAffinity:
operator: NotIn
values:
- {{ .Values.affinity.invokerNodeLabel }}
-# prefer to run on a control-plane node
+# prefer to run on a core node
nodeAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 80
@@ -21,7 +21,32 @@ nodeAffinity:
- key: openwhisk-role
operator: In
values:
- - {{ .Values.affinity.controlPlaneNodeLabel }}
+ - {{ .Values.affinity.coreNodeLabel }}
+{{- end -}}
+
+
+{{/* Generic edge affinity */}}
+{{- define "affinity.edge" -}}
+# prefer to not run on an invoker node (only prefer because of single node clusters)
+nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - {{ .Values.affinity.invokerNodeLabel }}
+# prefer to run on a edge node
+nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 80
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: In
+ values:
+ - {{ .Values.affinity.edgeNodeLabel }}
{{- end -}}
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
index 19dc021..917d311 100644
--- a/helm/templates/_helpers.tpl
+++ b/helm/templates/_helpers.tpl
@@ -1,28 +1,34 @@
-{{/* Generate kafka url without port */}}
-{{- define "kafka_url_without_port" -}}
-{{ .Values.kafka.name }}.{{ .Release.Namespace }}
+{{/* hostname for apigateway */}}
+{{- define "apigw_host" -}}
+{{ .Values.apigw.name }}.{{ .Release.Namespace }}.svc.cluster.local
{{- end -}}
-{{/* Generate Zookeeper service address */}}
-{{- define "zookeeper_service_address" -}}
-{{ .Values.zookeeper.name }}.{{ .Release.Namespace }}:{{ .Values.zookeeper.port }}
+{{/* hostname for controller */}}
+{{- define "controller_host" -}}
+{{ .Values.controller.name }}.{{ .Release.Namespace }}.svc.cluster.local
{{- end -}}
-{{/* Generate Couchdb url without port */}}
-{{- define "couchdb_url_without_port" -}}
-{{ .Values.db.name }}.{{ .Release.Namespace }}
+{{/* hostname for database */}}
+{{- define "db_host" -}}
+{{ .Values.db.name }}.{{ .Release.Namespace }}.svc.cluster.local
+{{- end -}}
+
+{{/* hostname for kafka */}}
+{{- define "kafka_host" -}}
+{{ .Values.kafka.name }}.{{ .Release.Namespace }}.svc.cluster.local
+{{- end -}}
+
+{{/* hostname for zookeeper */}}
+{{- define "zookeeper_host" -}}
+{{ .Values.zookeeper.name }}.{{ .Release.Namespace }}.svc.cluster.local
{{- end -}}
{{/* Runtimes manifest */}}
{{- define "runtimes_manifest" -}}
-{{- if .Values.travis -}}
-{{ .Files.Get "runtimes-minimal-travis.json" | quote }}
-{{- else -}}
-{{ .Files.Get "runtimes.json" | quote }}
-{{- end -}}
+{{ .Files.Get .Values.whisk.runtimes | quote }}
{{- end -}}
-{{/* Environment variables required for accessing CouchDB */}}
+{{/* Environment variables required for accessing CouchDB from a pod */}}
{{- define "whisk.dbEnvVars" -}}
- name: "CONFIG_whisk_couchdb_username"
value: {{ .Values.db.auth.username | quote }}
@@ -31,11 +37,11 @@
- name: "CONFIG_whisk_couchdb_port"
value: {{ .Values.db.port | quote}}
- name: "CONFIG_whisk_couchdb_protocol"
- value: "http"
+ value: {{ .Values.db.protocol | quote }}
- name: "CONFIG_whisk_couchdb_host"
- value: {{ include "couchdb_url_without_port" . | quote }}
+ value: {{ include "db_host" . | quote }}
- name: "CONFIG_whisk_couchdb_provider"
- value: "CouchDB"
+ value: {{ .Values.db.provider | quote }}
- name: "CONFIG_whisk_couchdb_databases_WhiskActivation"
value: {{ .Values.db.activationsTable | quote }}
- name: "CONFIG_whisk_couchdb_databases_WhiskEntity"
diff --git a/helm/templates/_readiness.tpl b/helm/templates/_readiness.tpl
index 5f1ca7f..f9e4174 100644
--- a/helm/templates/_readiness.tpl
+++ b/helm/templates/_readiness.tpl
@@ -5,7 +5,7 @@
imagePullPolicy: "IfNotPresent"
env:
- name: "READINESS_URL"
- value: http://{{ .Values.db.name }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.db.port }}/{{ .Values.db.activationsTable }}
+ value: {{ .Values.db.protocol }}://{{ include "db_host" . }}:{{ .Values.db.port }}/{{ .Values.db.activationsTable }}
command: ["sh", "-c", "result=1; until [ $result -eq 0 ]; do echo verifying CouchDB readiness; wget -T 5 --spider $READINESS_URL; result=$?; sleep 1; done;"]
{{- end -}}
@@ -15,7 +15,7 @@
image: "busybox"
imagePullPolicy: "IfNotPresent"
# TODO: I haven't found an easy external test to determine that kafka is up, so as a hack we wait for zookeeper and then sleep for 10 seconds and cross our fingers!
- command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ .Values.zookeeper.name}}.{{ .Release.Namespace }}.svc.cluster.local {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; fi; echo waiting for zookeeper to be ready; sleep 1; done; echo zookeeper is up, sleeping for 10 seconds; sleep 10;']
+ command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ include "zookeeper_host" . }} {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; fi; echo waiting for zookeeper to be ready; sleep 1; done; echo zookeeper is up, sleeping for 10 seconds; sleep 10;']
{{- end -}}
{{/* Init container that waits for zookeeper to be ready */}}
@@ -23,7 +23,7 @@
- name: "wait-for-zookeeper"
image: "busybox"
imagePullPolicy: "IfNotPresent"
- command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ .Values.zookeeper.name}}.{{ .Release.Namespace }}.svc.cluster.local {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; fi; echo waiting for zookeeper to be ready; sleep 1; done']
+ command: ["sh", "-c", 'result=1; until [ $result -eq 0 ]; do OK=$(echo ruok | nc -w 1 {{ include "zookeeper_host" . }} {{ .Values.zookeeper.port }}); if [ "$OK" == "imok" ]; then result=0; fi; echo waiting for zookeeper to be ready; sleep 1; done']
{{- end -}}
{{/* Init container that waits for controller to be ready */}}
@@ -33,6 +33,6 @@
imagePullPolicy: "IfNotPresent"
env:
- name: "READINESS_URL"
- value: http://{{ .Values.controller.name }}.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.controller.port }}/ping
+ value: http://{{ include "controller_host" . }}:{{ .Values.controller.port }}/ping
command: ["sh", "-c", "result=1; until [ $result -eq 0 ]; do echo verifying controller readiness; wget -T 5 --spider $READINESS_URL; result=$?; sleep 1; done;"]
{{- end -}}
diff --git a/helm/templates/apigateway.yaml b/helm/templates/apigateway.yaml
index 90c9870..6878eaf 100644
--- a/helm/templates/apigateway.yaml
+++ b/helm/templates/apigateway.yaml
@@ -7,13 +7,16 @@ metadata:
labels:
name: {{ .Values.apigw.name | quote }}
spec:
+ {{- if .Values.apigw.apiNodePort }}
+ type: NodePort
+ {{- end }}
selector:
name: {{ .Values.apigw.name | quote }}
ports:
- port: {{ .Values.apigw.mgmtPort }}
name: mgmt
- port: {{ .Values.apigw.apiPort }}
- {{- if .Values.apigw.apiNodePorthttpsNodePort }}
+ {{- if .Values.apigw.apiNodePort }}
nodePort: {{ .Values.apigw.apiNodePort }}
{{- end }}
name: api
@@ -37,7 +40,7 @@ spec:
{{- if .Values.affinity.enabled }}
affinity:
-{{ include "affinity.controlPlane" . | indent 8 }}
+{{ include "affinity.core" . | indent 8 }}
{{ include "affinity.selfAntiAffinity" ( .Values.apigw.name | quote ) | indent 8 }}
{{- end }}
diff --git a/helm/templates/controller.yaml b/helm/templates/controller.yaml
index 5f4ef8b..2c1e657 100644
--- a/helm/templates/controller.yaml
+++ b/helm/templates/controller.yaml
@@ -32,7 +32,7 @@ spec:
{{- if .Values.affinity.enabled }}
affinity:
-{{ include "affinity.controlPlane" . | indent 8 }}
+{{ include "affinity.core" . | indent 8 }}
{{ include "affinity.selfAntiAffinity" ( .Values.controller.name | quote ) | indent 8 }}
{{- end }}
@@ -84,7 +84,7 @@ spec:
# Kafka properties
- name: "KAFKA_HOSTS"
- value: {{ include "kafka_url_without_port" . }}:{{ .Values.kafka.port }}
+ value: "{{ include "kafka_host" . }}:{{ .Values.kafka.port }}"
- name: "KAFKA_HOST_PORT"
value: {{ .Values.kafka.port | quote }}
diff --git a/helm/templates/couchdb.yaml b/helm/templates/couchdb.yaml
index 7db28f2..268f9f6 100644
--- a/helm/templates/couchdb.yaml
+++ b/helm/templates/couchdb.yaml
@@ -75,7 +75,7 @@ spec:
{{- if .Values.affinity.enabled }}
affinity:
-{{ include "affinity.controlPlane" . | indent 8 }}
+{{ include "affinity.core" . | indent 8 }}
{{ include "affinity.selfAntiAffinity" ( .Values.db.name ) | indent 8 }}
{{- end }}
diff --git a/helm/templates/install_catalog_job.yaml b/helm/templates/install_catalog_job.yaml
index c494f7b..d845f4f 100644
--- a/helm/templates/install_catalog_job.yaml
+++ b/helm/templates/install_catalog_job.yaml
@@ -9,6 +9,7 @@ spec:
metadata:
name: install-catalog
spec:
+ restartPolicy: Never
initContainers:
# Wait for a controller to be up so we can perfom our CRUD actions with the CLI
{{ include "readiness.waitForController" . | indent 6 }}
@@ -28,4 +29,3 @@ spec:
configMapKeyRef:
name: whisk.ingress
key: api_host
- restartPolicy: Never
diff --git a/helm/templates/install_routemgmt_job.yaml b/helm/templates/install_routemgmt_job.yaml
new file mode 100644
index 0000000..b90d13f
--- /dev/null
+++ b/helm/templates/install_routemgmt_job.yaml
@@ -0,0 +1,35 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: install-routemgmt
+ namespace: {{ .Release.Namespace | quote }}
+spec:
+ activeDeadlineSeconds: 600
+ template:
+ metadata:
+ name: install-routemgmt
+ spec:
+ restartPolicy: Never
+ initContainers:
+ # Wait for a controller to be up so we can perfom our CRUD actions with the CLI
+{{ include "readiness.waitForController" . | indent 6 }}
+ containers:
+ - name: routemgmt
+ image: openwhisk/kube-routemgmt
+ env:
+ - name: "WHISK_CLI_VERSION"
+ value: {{ .Values.whisk.versions.cli | quote }}
+ - name: "WHISK_AUTH"
+ valueFrom:
+ secretKeyRef:
+ name: whisk.auth
+ key: system
+ - name: "WHISK_API_HOST_NAME"
+ valueFrom:
+ configMapKeyRef:
+ name: whisk.ingress
+ key: api_host
+ - name: "WHISK_NAMESPACE"
+ value: {{ .Values.whisk.systemNameSpace | quote }}
+ - name: "WHISK_API_GATEWAY_HOST_V2"
+ value: "http://$(APIGATEWAY_SERVICE_HOST):$(APIGATEWAY_SERVICE_PORT_API)/v2"
diff --git a/helm/templates/invoker.yaml b/helm/templates/invoker.yaml
index 5a83050..ed3e97a 100644
--- a/helm/templates/invoker.yaml
+++ b/helm/templates/invoker.yaml
@@ -103,11 +103,11 @@ spec:
# properties for kafka connection
- name: "KAFKA_HOSTS"
- value: "{{ .Values.kafka.name }}.{{ .Release.Namespace }}:{{ .Values.kafka.port }}"
+ value: "{{ include "kafka_host" . }}:{{ .Values.kafka.port }}"
# properties for zookeeper connection
- name: "ZOOKEEPER_HOSTS"
- value: "{{ .Values.zookeeper.name }}.{{ .Release.Namespace }}:{{ .Values.zookeeper.port }}"
+ value: "{{ include "zookeeper_host" . }}:{{ .Values.zookeeper.port }}"
ports:
- name: invoker
diff --git a/helm/templates/kafka.yaml b/helm/templates/kafka.yaml
index c2eb9cd..dc1a600 100644
--- a/helm/templates/kafka.yaml
+++ b/helm/templates/kafka.yaml
@@ -26,7 +26,7 @@ spec:
spec:
{{- if .Values.affinity.enabled }}
affinity:
-{{ include "affinity.controlPlane" . | indent 8 }}
+{{ include "affinity.core" . | indent 8 }}
{{ include "affinity.selfAntiAffinity" ( .Values.kafka.name | quote ) | indent 8 }}
{{- end }}
@@ -56,4 +56,4 @@ spec:
# zookeeper info
- name: "KAFKA_ZOOKEEPER_CONNECT"
- value: {{ include "zookeeper_service_address" . | quote }}
+ value: "{{ include "zookeeper_host" . }}:{{ .Values.zookeeper.port }}"
diff --git a/helm/templates/nginx.yaml b/helm/templates/nginx.yaml
index 5c3e28b..9110fb4 100644
--- a/helm/templates/nginx.yaml
+++ b/helm/templates/nginx.yaml
@@ -6,7 +6,9 @@ metadata:
labels:
name: {{ .Values.nginx.name | quote }}
spec:
+ {{- if .Values.nginx.httpsNodePort }}
type: NodePort
+ {{- end }}
selector:
name: {{ .Values.nginx.name | quote }}
ports:
@@ -39,7 +41,7 @@ spec:
{{- if .Values.affinity.enabled }}
affinity:
-{{ include "affinity.controlPlane" . | indent 8 }}
+{{ include "affinity.edge" . | indent 8 }}
{{ include "affinity.selfAntiAffinity" ( .Values.nginx.name | quote ) | indent 8 }}
{{- end }}
diff --git a/helm/templates/nginx_configmap.yaml b/helm/templates/nginx_configmap.yaml
index 1a596f2..0113cff 100644
--- a/helm/templates/nginx_configmap.yaml
+++ b/helm/templates/nginx_configmap.yaml
@@ -40,7 +40,7 @@ data:
# Hack to convince nginx to dynamically resolve the dns entry.
resolver kube-dns.kube-system;
- set $controllers controller.openwhisk.svc.cluster.local;
+ set $controllers {{ include "controller_host" . }};
# proxy to the web action path
location / {
diff --git a/helm/templates/zookeeper.yaml b/helm/templates/zookeeper.yaml
index 7a7f327..66b7f19 100644
--- a/helm/templates/zookeeper.yaml
+++ b/helm/templates/zookeeper.yaml
@@ -35,7 +35,7 @@ spec:
{{- if .Values.affinity.enabled }}
affinity:
-{{ include "affinity.controlPlane" . | indent 8 }}
+{{ include "affinity.core" . | indent 8 }}
{{ include "affinity.selfAntiAffinity" ( .Values.zookeeper.name | quote ) | indent 8 }}
{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
index 0c68b8b..c8cf6c2 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -13,8 +13,7 @@ whisk:
versions:
cli: "latest"
systemNameSpace: "/whisk.system"
-
-travis: false
+ runtimes: "runtimes.json"
# zookeeper configurations
zookeeper:
@@ -44,6 +43,8 @@ db:
imagePullPolicy: "IfNotPresent"
restartPolicy: "Always"
port: 5984
+ provider: "CouchDB"
+ protocol: "http"
auth:
username: "whisk_admin"
password: "some_passw0rd"
@@ -67,7 +68,6 @@ nginx:
restartPolicy: "Always"
httpPort: 80
httpsPort: 443
- httpsNodePort: 31001
httpsAdminPort: 8443
# Controller configurations
@@ -104,11 +104,12 @@ apigw:
imagePullPolicy: "IfNotPresent"
restartPolicy: "Always"
apiPort: 9000
- apiNodePort: 31004
mgmtPort: 8080
# Resolve the pod/node affinity for invoker, controller and other components.
affinity:
enabled: true
+ coreNodeLabel: core
+ edgeNodeLabel: edge
invokerNodeLabel: invoker
- controlPlaneNodeLabel: control-plane
+ providerNodeLabel: provider
diff --git a/tools/travis/build-helm.sh b/tools/travis/build-helm.sh
index 08d81eb..d5c6971 100755
--- a/tools/travis/build-helm.sh
+++ b/tools/travis/build-helm.sh
@@ -16,7 +16,8 @@ deploymentHealthCheck () {
TIMEOUT=0
until $PASSED || [ $TIMEOUT -eq $TIMEOUT_STEP_LIMIT ]; do
KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get pods -l name="$1" -o wide | grep "$1" | awk '{print $3}')
- if [ "$KUBE_DEPLOY_STATUS" == "Running" ]; then
+ KUBE_READY_COUNT=$(kubectl -n openwhisk get pods -l name="$1" -o wide | grep "$1" | awk '{print $2}' | awk -F / '${print $1}')
+ if [[ "$KUBE_DEPLOY_STATUS" == "Running" ]] && [[ "$KUBE_READY_COUNT" != "0" ]]; then
PASSED=true
break
fi
@@ -47,7 +48,8 @@ statefulsetHealthCheck () {
TIMEOUT=0
until $PASSED || [ $TIMEOUT -eq $TIMEOUT_STEP_LIMIT ]; do
KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get pods -l name="$1" -o wide | grep "$1"-0 | awk '{print $3}')
- if [ "$KUBE_DEPLOY_STATUS" == "Running" ]; then
+ KUBE_READY_COUNT=$(kubectl -n openwhisk get pods -l name="$1" -o wide | grep "$1"-0 | awk '{print $2}' | awk -F / '${print $1}')
+ if [[ "$KUBE_DEPLOY_STATUS" == "Running" ]] && [[ "$KUBE_READY_COUNT" != "0" ]]; then
PASSED=true
break
fi
@@ -132,7 +134,6 @@ kubectl create namespace openwhisk
# configure Ingress and wsk CLI
#
-# FIXME: Helm deploy hardwires ports to specific values -- need to make this less fragile!
WSK_PORT=31001
APIGW_PORT=31004
WSK_HOST=$(kubectl describe nodes | grep Hostname: | awk '{print $2}')
@@ -150,7 +151,14 @@ travis: true
whisk:
ingress:
api_host: $WSK_HOST:$WSK_PORT
- apigw_url: $WSK_HOST:$APIGW_PORT
+ apigw_url: http://$WSK_HOST:$APIGW_PORT
+ runtimes: "runtimes-minimal-travis.json"
+
+nginx:
+ httpsNodePort: $WSK_PORT
+
+apigw:
+ apiNodePort: $APIGW_PORT
EOF
cat mycluster.yaml
@@ -161,6 +169,10 @@ helm install . --namespace=openwhisk --name=ow4travis -f mycluster.yaml
statefulsetHealthCheck "controller"
deploymentHealthCheck "invoker"
+# Wait for catalog and routemgmt jobs to complete successfully
+jobHealthCheck "install-catalog"
+jobHealthCheck "install-routemgmt"
+
#################
# Sniff test: create and invoke a simple Hello world action
#################
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services