You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by mr...@apache.org on 2017/07/19 20:34:15 UTC
[incubator-openwhisk-deploy-kube] branch master updated: Manually
deploy the controller (#37)
This is an automated email from the ASF dual-hosted git repository.
mrutkowski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git
The following commit(s) were added to refs/heads/master by this push:
new 9319248 Manually deploy the controller (#37)
9319248 is described below
commit 9319248d4ae4c4090d653abe89d61fafaf3a1b50
Author: Dan Lavine <dl...@us.ibm.com>
AuthorDate: Wed Jul 19 20:34:13 2017 +0000
Manually deploy the controller (#37)
* Update Docs on how to deploy the controller
* Add a LivenessProbe to check that the controller's ping url is working:
This helps to know if the Controller process stops. (I.e. cannot connect to DB)
* Update CI
* Update cleanup script
---
.travis/build.sh | 45 +++++++-
README.md | 2 +-
.../environments/kube/files/controller-service.yml | 15 ---
.../environments/kube/files/controller.yml | 67 -----------
ansible-kube/openwhisk.yml | 2 -
ansible-kube/roles/controller/tasks/deploy.yml | 27 -----
configure/cleanup.sh | 2 +-
configure/configure.sh | 1 -
kubernetes/controller/README.md | 31 ++++++
kubernetes/controller/controller.yml | 124 +++++++++++++++++++++
kubernetes/nginx/nginx.conf | 12 +-
11 files changed, 204 insertions(+), 124 deletions(-)
diff --git a/.travis/build.sh b/.travis/build.sh
index 266787f..731f287 100755
--- a/.travis/build.sh
+++ b/.travis/build.sh
@@ -43,7 +43,7 @@ fi
echo "The job to configure OpenWhisk finished successfully"
-componentHealthCheck () {
+deploymentHealthCheck () {
if [ -z "$1" ]; then
echo "Error, component health check called without a component parameter"
exit 1
@@ -74,12 +74,51 @@ componentHealthCheck () {
echo "$1 is up and running"
}
+statefulsetHealthCheck () {
+ if [ -z "$1" ]; then
+ echo "Error, StatefulSet health check called without a parameter"
+ exit 1
+ fi
+
+ PASSED=false
+ TIMEOUT=0
+ until $PASSED || [ $TIMEOUT -eq 25 ]; do
+ KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get pods -o wide | grep "$1"-0 | awk '{print $3}')
+ if [ "$KUBE_DEPLOY_STATUS" == "Running" ]; then
+ PASSED=true
+ break
+ fi
+
+ kubectl get pods --all-namespaces -o wide --show-all
+
+ let TIMEOUT=TIMEOUT+1
+ sleep 30
+ done
+
+ if [ "$PASSED" = false ]; then
+ echo "Failed to finish deploying $1"
+
+ kubectl -n openwhisk logs $(kubectl -n openwhisk get pods -o wide | grep "$1"-0 | awk '{print $1}')
+ exit 1
+ fi
+
+ echo "$1-0 is up and running"
+
+}
+
+# setup the controller
+pushd kubernetes/controller
+ kubectl apply -f controller.yml
+
+ statefulsetHealthCheck "controller"
+popd
+
# setup the invoker
pushd kubernetes/invoker
kubectl apply -f invoker.yml
# wait until the invoker is ready
- componentHealthCheck "invoker"
+ statefulsetHealthCheck "invoker"
popd
# setup nginx
@@ -90,7 +129,7 @@ pushd kubernetes/nginx
kubectl apply -f nginx.yml
# wait until nginx is ready
- componentHealthCheck "nginx"
+ deploymentHealthCheck "nginx"
popd
AUTH_WSK_SECRET=789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
diff --git a/README.md b/README.md
index 03141b0..497ed42 100644
--- a/README.md
+++ b/README.md
@@ -97,6 +97,7 @@ kubectl -n openwhisk logs configure-openwhisk-XXXXX
Once the configuration job successfully finishes, you will need
manually deploy the rest of the OpenWhisk components.
+* [Controller](kubernetes/controller/README.md)
* [Invoker](kubernetes/invoker/README.md)
* [Nginx](kubernetes/nginx/README.md)
@@ -280,7 +281,6 @@ We also have experimental support for
## Enhancements
-* Allow users to provide custom certs for Nginx
* Enable the configuration job to run any number of times. This way it updates an already running
OpenWhisk deployment on all subsequent runs
* Use a public Edge Docker image once this [issue](https://github.com/apache/incubator-openwhisk/issues/2152)
diff --git a/ansible-kube/environments/kube/files/controller-service.yml b/ansible-kube/environments/kube/files/controller-service.yml
deleted file mode 100644
index b5762dd..0000000
--- a/ansible-kube/environments/kube/files/controller-service.yml
+++ /dev/null
@@ -1,15 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
- name: controller
- namespace: openwhisk
- labels:
- name: controller
-spec:
- selector:
- name: controller
- ports:
- - port: 8080
- targetPort: 8080
- name: controller
diff --git a/ansible-kube/environments/kube/files/controller.yml b/ansible-kube/environments/kube/files/controller.yml
deleted file mode 100644
index 53493c2..0000000
--- a/ansible-kube/environments/kube/files/controller.yml
+++ /dev/null
@@ -1,67 +0,0 @@
----
-apiVersion: extensions/v1beta1
-kind: Deployment
-metadata:
- name: controller
- namespace: openwhisk
- labels:
- name: controller
-spec:
- replicas: 1
- template:
- metadata:
- labels:
- name: controller
- spec:
- restartPolicy: Always
-
- containers:
- - name: controller
- imagePullPolicy: Always
- image: openwhisk/controller
- command: ["/bin/bash", "-c", "/controller/bin/controller 0"]
- ports:
- - name: controller
- containerPort: 8080
- env:
- - name: "COMPONENT_NAME"
- value: "controller"
- - name: "CONSULSERVER_HOST"
- value: "consul.openwhisk"
- - name: "CONSUL_HOST_PORT4"
- value: "8500"
- - name: "KAFKA_NUMPARTITIONS"
- value: "2"
- - name: "SERVICE_CHECK_HTTP"
- value: "/ping"
- - name: "SERVICE_CHECK_TIMEOUT"
- value: "2s"
- - name: "SERVICE_CHECK_INTERVAL"
- value: "15s"
- - name: "PORT"
- value: "8080"
- - name: "WHISK_VERSION_NAME"
- valueFrom:
- configMapKeyRef:
- name: controller
- key: whisk.version.name
- - name: "WHISK_VERSION_DATE"
- valueFrom:
- configMapKeyRef:
- name: controller
- key: whisk.version.date
- - name: "WHISK_VERSION_BUILDNO"
- valueFrom:
- configMapKeyRef:
- name: controller
- key: whisk.version.buildno
- - name: "JAVA_OPTS"
- valueFrom:
- configMapKeyRef:
- name: controller
- key: java.opts
- - name: "CONTROLLER_OPTS"
- valueFrom:
- configMapKeyRef:
- name: controller
- key: controller.opts
diff --git a/ansible-kube/openwhisk.yml b/ansible-kube/openwhisk.yml
index 0d74ad4..f45264e 100644
--- a/ansible-kube/openwhisk.yml
+++ b/ansible-kube/openwhisk.yml
@@ -7,6 +7,4 @@
- include: kafka.yml
-- include: controller.yml
-
#- include: routemgmt.yml
diff --git a/ansible-kube/roles/controller/tasks/deploy.yml b/ansible-kube/roles/controller/tasks/deploy.yml
deleted file mode 100644
index 43c7407..0000000
--- a/ansible-kube/roles/controller/tasks/deploy.yml
+++ /dev/null
@@ -1,27 +0,0 @@
----
-# This role will install Controller in group 'controllers' in the environment inventory
-- name: create controller config map
- shell: "kubectl create configmap controller --from-literal=whisk.version.name={{whisk_version_name}} --from-literal=whisk.version.date={{whisk.version.date}} --from-literal=whisk.version.buildno={{docker_image_tag}} --from-literal=java.opts=-Xmx{{controller.heap}} --from-literal=controller.opts={{controller.arguments}}"
-
-- name: create controller deployment
- shell: "kubectl apply -f {{kube_pod_dir}}/controller.yml"
-
-- name: get controller pods
- shell: "kubectl -n openwhisk get pods --show-all | grep controller | awk '{print $1}'"
- register: pods
- until: pods.stdout != ""
- retries: 5
- delay: 2
-
-- name: set controller pods
- set_fact:
- controller_pods: "{{ pods.stdout_lines }}"
-
-- name: wait until the Controller in this host is up and running
- shell: "kubectl -n openwhisk exec {{ item[0] }} -- bash -c 'curl -I http://0.0.0.0:8080/ping'"
- register: result
- until: (result.rc == 0) and (result.stdout.find("200 OK") != -1)
- retries: 20
- delay: 10
- with_items:
- - ["{{ controller_pods }}"]
diff --git a/configure/cleanup.sh b/configure/cleanup.sh
index 0c71f36..42310e5 100755
--- a/configure/cleanup.sh
+++ b/configure/cleanup.sh
@@ -12,7 +12,7 @@ kubectl -n openwhisk delete deployment couchdb
kubectl -n openwhisk delete deployment consul
kubectl -n openwhisk delete deployment zookeeper
kubectl -n openwhisk delete deployment kafka
-kubectl -n openwhisk delete deployment controller
+kubectl -n openwhisk delete statefulsets controller
kubectl -n openwhisk delete statefulsets invoker
kubectl -n openwhisk delete deployment nginx
diff --git a/configure/configure.sh b/configure/configure.sh
index 84bfadc..395a4c9 100755
--- a/configure/configure.sh
+++ b/configure/configure.sh
@@ -40,7 +40,6 @@ pushd /incubator-openwhisk-deploy-kube/ansible
kubectl apply -f environments/kube/files/consul-service.yml
kubectl apply -f environments/kube/files/zookeeper-service.yml
kubectl apply -f environments/kube/files/kafka-service.yml
- kubectl apply -f environments/kube/files/controller-service.yml
if deployCouchDB; then
# Create and configure the CouchDB deployment
diff --git a/kubernetes/controller/README.md b/kubernetes/controller/README.md
new file mode 100644
index 0000000..fee25c5
--- /dev/null
+++ b/kubernetes/controller/README.md
@@ -0,0 +1,31 @@
+Controller
+----------
+
+# Deploying
+
+When deploying the Controller, it needs to be deployed via a
+[StatefulSet][StatefulSet]. This is because each Controller
+instance needs to know which index it is. The Controller
+can be deployed with:
+
+```
+kubectl apply -f invoker.yml
+```
+
+# Controller Deployment Changes
+## Increase Controller Count
+
+If you want to increase the number of controllers deployed,
+then you will also need to update part of the Nginx configuration.
+First, you will need to update the replication count for the
+Controllers [here](https://github.com/apache/incubator-openwhisk-deploy-kube/tree/master/kubernetes/controller/controller.yml#L10).
+
+After updating the controller count, you will need to update
+the available routes for Nginx. This is because the controllers
+are not yet purely HA, but are in a failover mode. To update Nginx
+with the proper routes, take a look at
+[these properties](https://github.com/apache/incubator-openwhisk-deploy-kube/tree/master/kubernetes/nginx/nginx.conf#L15-L20).
+The routes for the controllers and how they are determined can
+be found in the [StatefulSet][StatefulSet] docs.
+
+[StatefulSet]: https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/
diff --git a/kubernetes/controller/controller.yml b/kubernetes/controller/controller.yml
new file mode 100644
index 0000000..f91ef34
--- /dev/null
+++ b/kubernetes/controller/controller.yml
@@ -0,0 +1,124 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: controller
+ namespace: openwhisk
+ labels:
+ name: controller
+spec:
+ selector:
+ name: controller
+ clusterIP: None
+ ports:
+ - port: 8080
+ targetPort: 8080
+ name: http
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+ name: controller
+ namespace: openwhisk
+ labels:
+ name: controller
+spec:
+ replicas: 2
+ serviceName: "controller"
+ template:
+ metadata:
+ labels:
+ name: controller
+ spec:
+ restartPolicy: Always
+
+ containers:
+ - name: controller
+ imagePullPolicy: Always
+ # Update this image to the publix OpenWhisk Image once this PR is merged.
+ # https://github.com/apache/incubator-openwhisk/pull/2452
+ image: danlavine/whisk_controller
+ command: ["/bin/bash", "-c", "/controller/bin/controller `hostname | cut -d'-' -f2`"]
+ ports:
+ - name: controller
+ containerPort: 8080
+ livenessProbe:
+ httpGet:
+ path: "/ping"
+ port: 8080
+ scheme: "HTTP"
+ initialDelaySeconds: 5
+ periodSeconds: 10
+ timeoutSeconds: 1
+ env:
+ - name: "PORT"
+ value: "8080"
+ - name: "COMPONENT_NAME"
+ value: "controller"
+ - name: "KAFKA_NUMPARTITIONS"
+ value: "2"
+ - name: "SERVICE_CHECK_HTTP"
+ value: "/ping"
+ - name: "SERVICE_CHECK_TIMEOUT"
+ value: "2s"
+ - name: "SERVICE_CHECK_INTERVAL"
+ value: "15s"
+
+ # Loadbalancer options
+ - name: "LOADBALANCER_INVOKERBUSYTHRESHOLD"
+ value: "16"
+
+ # This needs to stay up to date with the lates runtime in Ansible Groupvars
+ - name: "RUNTIMES_MANIFEST"
+ value: '{ "defaultImagePrefix": "openwhisk", "defaultImageTag": "latest", "runtimes": { "nodejs": [ { "kind": "nodejs", "image": { "name": "nodejsaction" }, "deprecated": true }, { "kind": "nodejs:6", "default": true, "image": { "name": "nodejs6action" }, "deprecated": false } ], "python": [ { "kind": "python", "image": { "name": "python2action" }, "deprecated": false }, { "kind": "python:2", "default": true, "image": { "name": "python2action" }, "deprecated": false }, { "kind" [...]
+
+ # this version is the day it is deployed and should be configured every time
+ - name: "WHISK_VERSION_DATE"
+ value: "2017-01-01T00:00:00Z"
+ # the buildno should be the docker image tag to use
+ - name: "WHISK_VERSION_BUILDNO"
+ value: "latest"
+
+ # Java options
+ - name: "JAVA_OPTS"
+ value: "-Xmx2g"
+
+ # Kafka properties
+ - name: "KAFKA_HOST"
+ value: "kafka.openwhisk"
+ - name: "KAFKA_HOST_PORT"
+ value: "9092"
+
+ # specific controller arguments
+ - name: "CONTROLLER_OPTS"
+ value: ""
+ - name: "DEFAULTLIMITS_ACTIONS_INVOKES_PERMINUTE"
+ value: "120"
+ - name: "DEFAULTLIMITS_ACTIONS_INVOKES_CONCURRENT"
+ value: "100"
+ - name: "DEFAULTLIMITS_ACTIONS_INVOKES_CONCURRENTINSYSTEM"
+ value: "500"
+ - name: "DEFAULTLIMITS_ACTIONS_SEQUENCE_MAXLENGTH"
+ value: "50"
+ - name: "DEFAULTLIMITS_TRIGGERS_FIRES_PERMINUTE"
+ value: "60"
+
+ # properties for DB connection
+ - name: "DB_USERNAME"
+ value: "couch_user"
+ - name: "DB_PASSWORD"
+ value: "couch_password"
+ - name: "DB_PORT"
+ value: "5984"
+ - name: "DB_PROTOCOL"
+ value: "http"
+ - name: "DB_HOST"
+ value: "couchdb.openwhisk"
+ - name: "DB_PROVIDER"
+ value: "CouchDB"
+ - name: "DB_WHISK_ACTIVATIONS"
+ value: "openwhisk_kube_activations"
+ - name: "DB_WHISK_ACTIONS"
+ value: "openwhisk_kube_whisks"
+ - name: "DB_WHISK_AUTHS"
+ value: "openwhisk_kube_subjects"
diff --git a/kubernetes/nginx/nginx.conf b/kubernetes/nginx/nginx.conf
index a24e689..c76c40d 100644
--- a/kubernetes/nginx/nginx.conf
+++ b/kubernetes/nginx/nginx.conf
@@ -12,14 +12,12 @@ http {
access_log /logs/nginx_access.log combined-upstream;
upstream controllers {
- server controller.openwhisk:8080 fail_timeout=60s;
+ # These addresses are for the controller. If there are more
+ # than two invoker instances deployed then you will need to
+ # add additional controller addresses.
- # TODO: Remove the above controller setup and remove the commented
- # lines below once the Controller has ben converted to a pure yaml
- # configuration.
- #
- # server controller-0.openwhisk:8080 fail_timeout=60s;
- # server controller-1.openwhisk:8080 backup;
+ server controller-0.controller.openwhisk:8080 fail_timeout=60s;
+ server controller-1.controller.openwhisk:8080 backup;
}
server {
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].