You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by mr...@apache.org on 2017/07/14 16:41:27 UTC

[incubator-openwhisk-deploy-kube] branch master updated: Deploy Invoker manually (#34)

This is an automated email from the ASF dual-hosted git repository.

mrutkowski pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git


The following commit(s) were added to refs/heads/master by this push:
     new 5f03b2f  Deploy Invoker manually (#34)
5f03b2f is described below

commit 5f03b2f7a25a1acb51d5214ca7e6576c8155c374
Author: Dan Lavine <dl...@us.ibm.com>
AuthorDate: Fri Jul 14 16:41:25 2017 +0000

    Deploy Invoker manually (#34)
    
    * Provide docs for how to deploy Invoker
    * Update travis
    * Update cleanup script
---
 .travis/build.sh                                   |  62 ++++++---
 README.md                                          |  30 +----
 .../environments/kube/files/invoker-service.yml    |  16 ---
 ansible-kube/environments/kube/files/invoker.yml   |  76 -----------
 ansible-kube/environments/kube/group_vars/all      |   5 -
 ansible-kube/openwhisk.yml                         |   2 -
 ansible-kube/roles/invoker/tasks/deploy.yml        |  15 ---
 ansible-kube/roles/kafka/tasks/deploy.yml          |  17 ---
 ansible-kube/tasks/writeWhiskProperties.yml        |   9 --
 ansible-kube/templates/whisk.properties.j2         |   7 --
 configure/cleanup.sh                               |   1 -
 configure/configure.sh                             |  10 --
 kubernetes/invoker/README.md                       |  58 +++++++++
 kubernetes/invoker/invoker.yml                     | 140 +++++++++++++++++++++
 14 files changed, 245 insertions(+), 203 deletions(-)

diff --git a/.travis/build.sh b/.travis/build.sh
index fa5df1c..266787f 100755
--- a/.travis/build.sh
+++ b/.travis/build.sh
@@ -7,9 +7,9 @@ ROOTDIR="$SCRIPTDIR/../"
 
 cd $ROOTDIR
 
+# setup the appropriate configuration image
 sed -ie "s/whisk_config:v1.5.6/whisk_config:$TRAVIS_KUBE_VERSION/g" configure/configure_whisk.yml
 
-# run scripts to deploy using the new images.
 kubectl apply -f configure/openwhisk_kube_namespace.yml
 kubectl apply -f configure/configure_whisk.yml
 
@@ -43,34 +43,59 @@ fi
 
 echo "The job to configure OpenWhisk finished successfully"
 
-# setup nginx
-pushd kubernetes/nginx
-  ./certs.sh localhost
-  kubectl -n openwhisk create configmap nginx --from-file=nginx.conf
-  kubectl -n openwhisk create secret tls nginx --cert=certs/cert.pem --key=certs/key.pem
-  kubectl apply -f nginx.yml
-
-  WSK_PORT=$(kubectl -n openwhisk describe service nginx | grep https-api | grep NodePort| awk '{print $3}' | cut -d'/' -f1)
+componentHealthCheck () {
+  if [ -z "$1" ]; then
+    echo "Error, component health check called without a component parameter"
+    exit 1
+  fi
 
-  # wait untill nginx is ready
+  PASSED=false
   TIMEOUT=0
-  TIMEOUT_COUNT=40
-  until $(curl --output /dev/null --silent -k https://localhost:$WSK_PORT) || [ $TIMEOUT -eq $TIMEOUT_COUNT ]; do
-    echo "Nginx is not up yet"
+  until $PASSED || [ $TIMEOUT -eq 25 ]; do
+    KUBE_DEPLOY_STATUS=$(kubectl -n openwhisk get pods -o wide | grep "$1" | awk '{print $3}')
+    if [ "$KUBE_DEPLOY_STATUS" == "Running" ]; then
+      PASSED=true
+      break
+    fi
+
+    kubectl get pods --all-namespaces -o wide --show-all
+
     let TIMEOUT=TIMEOUT+1
-    sleep 20
+    sleep 30
   done
 
-  if [ $TIMEOUT -eq $TIMEOUT_COUNT ]; then
-    echo "Nginx is not up and running"
+  if [ "$PASSED" = false ]; then
+    echo "Failed to finish deploying $1"
+
+    kubectl -n openwhisk logs $(kubectl -n openwhisk get pods -o wide | grep "$1" | awk '{print $1}')
     exit 1
   fi
+
+  echo "$1 is up and running"
+}
+
+# setup the invoker
+pushd kubernetes/invoker
+  kubectl apply -f invoker.yml
+
+  # wait until the invoker is ready
+  componentHealthCheck "invoker"
 popd
 
-echo "Nginx is up and running"
+# setup nginx
+pushd kubernetes/nginx
+  ./certs.sh localhost
+  kubectl -n openwhisk create configmap nginx --from-file=nginx.conf
+  kubectl -n openwhisk create secret tls nginx --cert=certs/cert.pem --key=certs/key.pem
+  kubectl apply -f nginx.yml
+
+  # wait until nginx is ready
+  componentHealthCheck "nginx"
+popd
 
 AUTH_WSK_SECRET=789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
 AUTH_GUEST=23bc46b1-71f6-4ed5-8c54-816aa4f8c502:123zO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP
+WSK_PORT=$(kubectl -n openwhisk describe service nginx | grep https-api | grep NodePort| awk '{print $3}' | cut -d'/' -f1)
 
 # download and setup the wsk cli from nginx
 wget --no-check-certificate https://localhost:$WSK_PORT/cli/go/download/linux/amd64/wsk
@@ -104,7 +129,6 @@ EOL
 
 ./wsk -i action create hello hello.js
 
-
 sleep 5
 
 # run the new hello world action
@@ -113,9 +137,9 @@ RESULT=$(./wsk -i action invoke --blocking hello | grep "\"status\": \"success\"
 if [ -z "$RESULT" ]; then
   echo "FAILED! Could not invoked custom action"
 
-
   echo " ----------------------------- controller logs ---------------------------"
   kubectl -n openwhisk logs $(kubectl get pods --all-namespaces -o wide | grep controller | awk '{print $2}')
+
   echo " ----------------------------- invoker logs ---------------------------"
   kubectl -n openwhisk logs $(kubectl get pods --all-namespaces -o wide | grep invoker | awk '{print $2}')
   exit 1
diff --git a/README.md b/README.md
index 9d313ac..03141b0 100644
--- a/README.md
+++ b/README.md
@@ -96,8 +96,9 @@ kubectl -n openwhisk logs configure-openwhisk-XXXXX
 ```
 
 Once the configuration job successfully finishes, you will need
-manually deploy Nginx. To do this, follow the Nginx
-[README](kubernetes/nginx/README.md).
+manually deploy the rest of the OpenWhisk components.
+* [Invoker](kubernetes/invoker/README.md)
+* [Nginx](kubernetes/nginx/README.md)
 
 From here, you will now need to get the publicly available address
 of Nginx.
@@ -139,7 +140,7 @@ replace the `[nginx_ip]` bellow.
 ```
   pushd /tmp
     git clone https://github.com/apache/incubator-openwhisk-catalog
-    cd incubator-openwhisk-catalog
+    cd incubator-openwhisk-catalog/packages
     ./installCatalog.sh 789c46b1-71f6-4ed5-8c54-816aa4f8c502:abczO3xZCLrMN6v2BKK1dXYFpXlPkccOFqm12CdAsMgRU4VrNZ9lyGVCGuMDGIwP https://[nginx_ip]:$WSK_PORT
   popd
 ```
@@ -175,24 +176,6 @@ in the Kube cluster. You will need to create a ClusterRoleBinding with proper
 security settings. For information about the role bindings,
 take a look at the info [here](https://kubernetes.io/docs/admin/authorization/rbac/).
 
-## Kubernetes Host Linux Versions
-
-Unfortunitaly when Deploying OpenWhisk on Kubernetes it currently mounts some
-of the host OS files for the Invoker process and needs to make some assumptions.
-Because of this, some failures are known to happen on certain Linux versions,
-like CoreOs. If you see an error like:
-
-```
-Failed to start container with id 8d9125bf2d3711312a98a8b98de15306e495883cc470a03beb6689b34895791f with error: rpc error: code = 2 desc = failed to start container "8d9125bf2d3711312a98a8b98de15306e495883cc470a03beb6689b34895791f": Error response from daemon: {"message":"mkdir /usr/lib/x86_64-linux-gnu: read-only file system"}
-Error syncing pod, skipping: failed to "StartContainer" for "invoker" with rpc error: code = 2 desc = failed to start container "8d9125bf2d3711312a98a8b98de15306e495883cc470a03beb6689b34895791f": Error response from daemon: {"message":"mkdir /usr/lib/x86_64-linux-gnu: read-only file system"}: "Start Container Failed"
-```
-
-Then you might need to modify some of the volume mounts in the
-[invoker.yml](ansible-kube/environments/kube/files/invoker.yml). For example,
-the error above is trying to find something from the apparmor mount which makes no
-sense to CoreOS. To fix the issue, you will need to remove the mount and rebuild
-the [custom Docker image](#manually-building-custom-docker-files).
-
 # Manually Building Custom Docker Files
 
 There are two images that are required when deploying OpenWhisk on Kube,
@@ -267,11 +250,6 @@ for Nginx and currently it has a static dns entry. Because of this, you
 will need to connect to OpenWhisk using the insecure mode (e.g. `wsk -i`).
 There is future work to make this CA-cert configurable.
 
-For now, OpenWhisk relies on part of the underlying infrastructure that Kube
-is running on. When deploying the Invoker for OpenWhisk, it mounts the hosts
-Docker socket. This way OpenWhisk can quickly provision actions and does not
-have to run Docker inside of Docker.
-
 A couple of components for OpenWhisk on Kube deployment strategy requires custom
 built Docker images. One such component is Nginx and currently resides at
 [danlavine/whisk_nginx](https://hub.docker.com/r/danlavine/whisk_nginx/). There
diff --git a/ansible-kube/environments/kube/files/invoker-service.yml b/ansible-kube/environments/kube/files/invoker-service.yml
deleted file mode 100644
index 9c0e093..0000000
--- a/ansible-kube/environments/kube/files/invoker-service.yml
+++ /dev/null
@@ -1,16 +0,0 @@
----
-apiVersion: v1
-kind: Service
-metadata:
-  name: invoker
-  namespace: openwhisk
-  labels:
-    name: invoker
-spec:
-  selector:
-    name: invoker
-  clusterIP: None
-  ports:
-    - port: 8080
-      targetPort: 8080
-      name: invoker
diff --git a/ansible-kube/environments/kube/files/invoker.yml b/ansible-kube/environments/kube/files/invoker.yml
deleted file mode 100644
index 7ae579e..0000000
--- a/ansible-kube/environments/kube/files/invoker.yml
+++ /dev/null
@@ -1,76 +0,0 @@
----
-apiVersion: apps/v1beta1
-kind: StatefulSet
-metadata:
-  name: invoker
-  namespace: openwhisk
-  labels:
-    name: invoker
-spec:
-  replicas: 1
-  serviceName: "invoker"
-  template:
-    metadata:
-      labels:
-        name: invoker
-    spec:
-      restartPolicy: Always
-
-      volumes:
-      - name: cgroup
-        hostPath:
-          path: "/sys/fs/cgroup"
-      - name: runc
-        hostPath:
-          path: "/run/runc"
-      - name: dockerrootdir
-        hostPath:
-          path: "/var/lib/docker/containers"
-      - name: dockersock
-        hostPath:
-          path: "/var/run/docker.sock"
-      - name: apparmor
-        hostPath:
-          path: "/usr/lib/x86_64-linux-gnu/libapparmor.so.1"
-
-      containers:
-      - name: invoker
-        imagePullPolicy: Always
-        image: openwhisk/invoker
-        command: [ "/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /invoker/bin/invoker `hostname | cut -d'-' -f2`" ]
-        env:
-          - name: "CONSULSERVER_HOST"
-            value: "consul.openwhisk"
-          - name: "CONSUL_HOST_PORT4"
-            value: "8500"
-          - name: "PORT"
-            value: "8080"
-          - name: "SELF_DOCKER_ENDPOINT"
-            value: "localhost"
-          - name: "SERVICE_CHECK_HTTP"
-            value: "/ping"
-          - name: "SERVICE_CHECK_TIMEOUT"
-            value: "2s"
-          - name: "SERVICE_CHECK_INTERVAL"
-            value: "15s"
-        ports:
-        - name: invoker
-          containerPort: 8080
-        volumeMounts:
-        - name: cgroup
-          mountPath: "/sys/fs/cgroup"
-        - name: runc
-          mountPath: "/run/runc"
-        - name: dockersock
-          mountPath: "/var/run/docker.sock"
-        - name: dockerrootdir
-          mountPath: "/containers"
-        - name: apparmor
-          mountPath: "/usr/lib/x86_64-linux-gnu/libapparmor.so.1"
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - "/bin/bash"
-              - "-c"
-              - "docker pull openwhisk/nodejs6action && docker pull openwhisk/dockerskeleton && docker pull openwhisk/python2action && docker pull openwhisk/python3action && docker pull openwhisk/swift3action && docker pull openwhisk/java8action"
diff --git a/ansible-kube/environments/kube/group_vars/all b/ansible-kube/environments/kube/group_vars/all
index 934a1e6..bc7b59e 100644
--- a/ansible-kube/environments/kube/group_vars/all
+++ b/ansible-kube/environments/kube/group_vars/all
@@ -52,11 +52,6 @@ kafka_host: kafka.openwhisk
 zookeeper_host: zookeeper.openwhisk
 
 # invoker properties
-# The invoker_count property is overwritten by the (kubernetes/configure/configure.sh)
-# script. This way the source of truth for the number of Invoker instances is kept
-# in the Kubernetes Invoker deployment file. The configure.sh script will read the
-# Kube file and replace this one to keep everything in sync.
-invoker_count: REPLACE_INVOKER_COUNT
 invoker_port: 8080
 
 # registry
diff --git a/ansible-kube/openwhisk.yml b/ansible-kube/openwhisk.yml
index 0aed6a0..0d74ad4 100644
--- a/ansible-kube/openwhisk.yml
+++ b/ansible-kube/openwhisk.yml
@@ -9,6 +9,4 @@
 
 - include: controller.yml
 
-- include: invoker.yml
-
 #- include: routemgmt.yml
diff --git a/ansible-kube/roles/invoker/tasks/deploy.yml b/ansible-kube/roles/invoker/tasks/deploy.yml
deleted file mode 100644
index aea853b..0000000
--- a/ansible-kube/roles/invoker/tasks/deploy.yml
+++ /dev/null
@@ -1,15 +0,0 @@
----
-# This role installs invokers.
-- name: create invoker deployment
-  shell: "kubectl apply -f {{kube_pod_dir}}/invoker.yml"
-
-# The invoker image has a long pull timout since it needs to pull
-# all of the Docker images whisk depends on.
-- name: wait until Invoker is up and running
-  uri:
-    url: "http://invoker-{{ item }}.invoker.openwhisk:{{ invoker_port }}/ping"
-  register: result
-  until: result.status == 200
-  retries: 50
-  delay: 20
-  with_sequence: start=0 count={{ invoker_count }}
diff --git a/ansible-kube/roles/kafka/tasks/deploy.yml b/ansible-kube/roles/kafka/tasks/deploy.yml
index 8a4a321..9db4f65 100644
--- a/ansible-kube/roles/kafka/tasks/deploy.yml
+++ b/ansible-kube/roles/kafka/tasks/deploy.yml
@@ -53,20 +53,3 @@
   with_nested:
   - "{{ kafka_pods }}"
   - [ 'command', 'health' ]
-
-- name: define invoker list
-  set_fact:
-    invoker_list: []
-
-- name: create the invoker list
-  set_fact:
-    invoker_list: "{{invoker_list}} + [{{item}}]"
-  with_sequence: start=0 count={{ invoker_count }}
-
-- name: create the invoker topics
-  shell: " kubectl exec {{ item[0] }} -c kafka -- bash -c 'unset JMX_PORT; kafka-topics.sh --create --topic invoker{{ item[1] }} --replication-factor 1 --partitions 1 --zookeeper {{ zookeeper_host }}:{{ zookeeper.port }}'"
-  register: command_result
-  failed_when: "not ('Created topic' in command_result.stdout or 'already exists' in command_result.stdout)"
-  with_nested:
-  - "{{ kafka_pods }}"
-  - "{{ invoker_list }}"
diff --git a/ansible-kube/tasks/writeWhiskProperties.yml b/ansible-kube/tasks/writeWhiskProperties.yml
index d066c86..42a29b0 100644
--- a/ansible-kube/tasks/writeWhiskProperties.yml
+++ b/ansible-kube/tasks/writeWhiskProperties.yml
@@ -2,15 +2,6 @@
 # This task will write whisk.properties to the openwhisk_home.
 # Currently whisk.properties is still needed for consul and tests.
 
-- name: define invoker domains
-  set_fact:
-    invoker_hosts: []
-
-- name: update the invoker domains
-  set_fact:
-    invoker_hosts: "{{ invoker_hosts }} + [ 'invoker-{{item}}.invoker.openwhisk' ]"
-  with_sequence: start=0 count={{ invoker_count }}
-
 - name: write whisk.properties template to openwhisk_home
   template:
     src: whisk.properties.j2
diff --git a/ansible-kube/templates/whisk.properties.j2 b/ansible-kube/templates/whisk.properties.j2
index b49c8a4..18be01f 100644
--- a/ansible-kube/templates/whisk.properties.j2
+++ b/ansible-kube/templates/whisk.properties.j2
@@ -2,7 +2,6 @@ openwhisk.home={{ openwhisk_home }}
 
 python.27=python
 use.cli.download=false
-nginx.conf.dir={{ nginx_conf_dir }}
 testing.auth={{ openwhisk_home }}/ansible/files/auth.guest
 vcap.services.file=
 
@@ -50,12 +49,9 @@ limits.triggers.fires.perMinute={{ limits.triggers.fires.perMinute }}
 
 # DNS host resolution
 consulserver.host={{ consul_host }}
-invoker.hosts={{ invoker_hosts | join(",") }}
 kafka.host={{ kafka_host }}
 zookeeper.host={{ zookeeper_host }}
-edge.host={{ nginx_host }}
 loadbalancer.host={{ controller_host }}
-router.host={{ nginx_host }}
 
 {#
  # replaced host entries
@@ -105,9 +101,6 @@ invoker.serializeDockerPull={{ invoker.serializeDockerPull }}
 invoker.useRunc={{ invoker_use_runc | default(invoker.useRunc) }}
 invoker.useReactivePool={{ invoker.useReactivePool }}
 
-invoker.instances={{ invoker_count }}
-#invoker.instances={{ invoker.instances }}
-
 consulserver.docker.endpoint={{ groups["consul_servers"]|first }}:{{ docker.port }}
 edge.docker.endpoint={{ groups["edge"]|first }}:{{ docker.port }}
 kafka.docker.endpoint={{ groups["kafka"]|first }}:{{ docker.port }}
diff --git a/configure/cleanup.sh b/configure/cleanup.sh
index 360482d..0c71f36 100755
--- a/configure/cleanup.sh
+++ b/configure/cleanup.sh
@@ -30,5 +30,4 @@ kubectl -n openwhisk delete service consul
 kubectl -n openwhisk delete service zookeeper
 kubectl -n openwhisk delete service kafka
 kubectl -n openwhisk delete service controller
-kubectl -n openwhisk delete service invoker
 kubectl -n openwhisk delete service nginx
diff --git a/configure/configure.sh b/configure/configure.sh
index e53d743..84bfadc 100755
--- a/configure/configure.sh
+++ b/configure/configure.sh
@@ -18,15 +18,6 @@ deployCouchDB() {
 
 set -ex
 
-# Currently, Consul needs to be seeded with the proper Invoker name to DNS address. To account for
-# this, we need to use StatefulSets(https://kubernetes.io/stutorials/stateful-application/basic-stateful-set/)
-# to generate the Invoker addresses in a guranteed pattern. We can then use properties from the
-# StatefulSet yaml file for OpenWhisk deployment configuration options.
-
-INVOKER_REP_COUNT=$(cat /incubator-openwhisk-deploy-kube/ansible-kube/environments/kube/files/invoker.yml | grep 'replicas:' | awk '{print $2}')
-INVOKER_COUNT=${INVOKER_REP_COUNT:-1}
-sed -ie "s/REPLACE_INVOKER_COUNT/$INVOKER_COUNT/g" /incubator-openwhisk-deploy-kube/ansible-kube/environments/kube/group_vars/all
-
 # copy the ansible playbooks and tools to this repo
 cp -R /openwhisk/ansible/ /incubator-openwhisk-deploy-kube/ansible
 cp -R /openwhisk/tools/ /incubator-openwhisk-deploy-kube/tools
@@ -50,7 +41,6 @@ pushd /incubator-openwhisk-deploy-kube/ansible
   kubectl apply -f environments/kube/files/zookeeper-service.yml
   kubectl apply -f environments/kube/files/kafka-service.yml
   kubectl apply -f environments/kube/files/controller-service.yml
-  kubectl apply -f environments/kube/files/invoker-service.yml
 
   if deployCouchDB; then
     # Create and configure the CouchDB deployment
diff --git a/kubernetes/invoker/README.md b/kubernetes/invoker/README.md
new file mode 100644
index 0000000..69308df
--- /dev/null
+++ b/kubernetes/invoker/README.md
@@ -0,0 +1,58 @@
+Invoker
+-------
+
+# Deploying
+
+When deploying the Invoker, it needs to be deployed via a
+[StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/).
+This is because each Invoker instance needs to know the instance
+it is for the Kafka topic. The current deployment is a single
+Invoker instance and can be deployed with:
+
+```
+kubectl apply -f invoker.yml
+```
+
+**Important**
+
+OpenWhisk relies on part of the underlying infrastructure that Kube
+is running on. When deploying the Invoker for OpenWhisk, it mounts the hosts
+Docker socket and a number of other components. This way OpenWhisk can
+quickly provision actions and does not have to run Docker inside of Docker.
+However, this also means that a number of the default mount options assume
+that the Kubernetes host image is Ubuntu. During the deploy there could be an
+issue and if the Invoker fails to deploy, see the [Troubleshooting](#troubleshooting)
+section below.
+
+# Invoker Deployment Changes
+## Increase Invoker Count
+
+To increase the number of Invokers, edit the
+[replicas](https://github.com/apache/incubator-openwhisk-deploy-kube/tree/master/kubernetes/invoker/invoker.yml#L27)
+line. Secondly, you will need to update the
+[INVOKER_COUNT](https://github.com/apache/incubator-openwhisk-deploy-kube/tree/master/kubernetes/invoker/invoker.yml#L63-L64)
+to with the same replica count.
+
+## Deploying Invoker to Specific Kube Nodes
+
+To deploy an Invoker to specific Kube nodes, you will need to edit the
+[invoker.yml](https://github.com/apache/incubator-openwhisk-deploy-kube/tree/master/kubernetes/invoker/invoker.yml)
+file with Kubernetes [NodeSelectors](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/).
+
+# Troubleshooting
+## Kubernetes Host Linux Versions
+
+Unfortunitaly when Deploying OpenWhisk on Kubernetes it currently mounts some
+of the host OS files for the Invoker process and needs to make some assumptions.
+Because of this, some failures are known to happen on certain Linux versions,
+like CoreOs. If you see an error like:
+
+```
+Failed to start container with id 8d9125bf2d3711312a98a8b98de15306e495883cc470a03beb6689b34895791f with error: rpc error: code = 2 desc = failed to start container "8d9125bf2d3711312a98a8b98de15306e495883cc470a03beb6689b34895791f": Error response from daemon: {"message":"mkdir /usr/lib/x86_64-linux-gnu: read-only file system"}
+Error syncing pod, skipping: failed to "StartContainer" for "Invoker" with rpc error: code = 2 desc = failed to start container "8d9125bf2d3711312a98a8b98de15306e495883cc470a03beb6689b34895791f": Error response from daemon: {"message":"mkdir /usr/lib/x86_64-linux-gnu: read-only file system"}: "Start Container Failed"
+```
+
+Then you might need to modify some of the volume mounts in the
+[invoker.yml](invoker.yml). For example,
+the error above is trying to find something from the apparmor mount which makes no
+sense to CoreOS. To fix the issue, you just need to remove the mount.
diff --git a/kubernetes/invoker/invoker.yml b/kubernetes/invoker/invoker.yml
new file mode 100644
index 0000000..fc7f79c
--- /dev/null
+++ b/kubernetes/invoker/invoker.yml
@@ -0,0 +1,140 @@
+---
+apiVersion: apps/v1beta1
+kind: StatefulSet
+metadata:
+  name: invoker
+  namespace: openwhisk
+  labels:
+    name: invoker
+spec:
+  replicas: 1
+  serviceName: "invoker"
+  template:
+    metadata:
+      labels:
+        name: invoker
+    spec:
+      restartPolicy: Always
+
+      volumes:
+      - name: cgroup
+        hostPath:
+          path: "/sys/fs/cgroup"
+      - name: runc
+        hostPath:
+          path: "/run/runc"
+      - name: dockerrootdir
+        hostPath:
+          path: "/var/lib/docker/containers"
+      - name: dockersock
+        hostPath:
+          path: "/var/run/docker.sock"
+      - name: apparmor
+        hostPath:
+          path: "/usr/lib/x86_64-linux-gnu/libapparmor.so.1"
+
+      containers:
+      - name: invoker
+        imagePullPolicy: Always
+        # TODO: change this to the public openwhisk image for nuking Consul.
+        # there is a PR that needs to be merged first.
+        # https://github.com/apache/incubator-openwhisk/pull/2452
+        image: danlavine/whisk_invoker
+        command: [ "/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /invoker/bin/invoker `hostname | cut -d'-' -f2`" ]
+        env:
+          - name: "PORT"
+            value: "8080"
+          - name: "SELF_DOCKER_ENDPOINT"
+            value: "localhost"
+          - name: "SERVICE_CHECK_HTTP"
+            value: "/ping"
+          - name: "SERVICE_CHECK_TIMEOUT"
+            value: "2s"
+          - name: "SERVICE_CHECK_INTERVAL"
+            value: "15s"
+          - name: "WHISK_API_HOST_NAME"
+            value: "nginx.openwhisk"
+          - name: "WHISK_VERSION_BUILDNO"
+            value: "latest"
+          - name: "INVOKER_CONTAINER_NETWORK"
+            value: "bridge"
+
+          # Properties for invoker image
+          - name: "DOCKER_IMAGE_PREFIX"
+            value: "openwhisk"
+          - name: "DOCKER_IMAGE_TAG"
+            value: "latest"
+          - name: "DOCKER_REGISTRY"
+            value: ""
+
+          # Invoker instance count. Needs to match replica count
+          - name: "INVOKER_INSTANCES"
+            value: "1"
+
+          # Java options
+          - name: "JAVA_OPTS"
+            value: "-Xmx2g"
+
+          # Invoker options
+          - name: "INVOKER_OPTS"
+            value: ""
+
+          # Kafka properties
+          - name: "KAFKA_HOST"
+            value: "kafka.openwhisk"
+          - name: "KAFKA_HOST_PORT"
+            value: "9092"
+
+          # This property can change since it is generated via Ansible GroupVars
+          - name: "RUNTIMES_MANIFEST"
+            value: '{ "defaultImagePrefix": "openwhisk", "defaultImageTag": "latest", "runtimes": { "nodejs": [ { "kind": "nodejs", "image": { "name": "nodejsaction" }, "deprecated": true }, { "kind": "nodejs:6", "default": true, "image": { "name": "nodejs6action" }, "deprecated": false } ], "python": [ { "kind": "python", "image": { "name": "python2action" }, "deprecated": false }, { "kind": "python:2", "default": true, "image": { "name": "python2action" }, "deprecated": false }, { "kin [...]
+
+          # Default to empty logs dir. This is because logs should go to stdout
+          - name: "WHISK_LOGS_DIR"
+            value: ""
+
+          # this version is the day it is deployed,
+          - name:  "WHISK_VERSION_DATE"
+            value: "2017-01-01T00:00:00Z"
+
+          # properties for DB connection
+          - name: "DB_USERNAME"
+            value: "couch_user"
+          - name: "DB_PASSWORD"
+            value: "couch_password"
+          - name: "DB_PORT"
+            value: "5984"
+          - name:  "DB_PROTOCOL"
+            value: "http"
+          - name: "DB_HOST"
+            value: "couchdb.openwhisk"
+          - name: "DB_PROVIDER"
+            value: "CouchDB"
+          - name: "DB_WHISK_ACTIVATIONS"
+            value: "openwhisk_kube_activations"
+          - name: "DB_WHISK_ACTIONS"
+            value: "openwhisk_kube_whisks"
+          - name: "DB_WHISK_AUTHS"
+            value: "openwhisk_kube_subjects"
+        ports:
+        - name: invoker
+          containerPort: 8080
+        volumeMounts:
+        - name: cgroup
+          mountPath: "/sys/fs/cgroup"
+        - name: runc
+          mountPath: "/run/runc"
+        - name: dockersock
+          mountPath: "/var/run/docker.sock"
+        - name: dockerrootdir
+          mountPath: "/containers"
+        - name: apparmor
+          mountPath: "/usr/lib/x86_64-linux-gnu/libapparmor.so.1"
+        lifecycle:
+          postStart:
+            exec:
+              command:
+              - "/bin/bash"
+              - "-c"
+              - "docker pull openwhisk/nodejsactionbase && docker pull openwhisk/nodejs6action && docker pull openwhisk/dockerskeleton && docker pull openwhisk/python2action && docker pull openwhisk/python3action && docker pull openwhisk/action-swift-v3.1.1 && docker pull openwhisk/swift3action && docker pull openwhisk/java8action"
+

-- 
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].