You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by cs...@apache.org on 2018/06/04 14:09:05 UTC

[incubator-openwhisk-deploy-kube] branch master updated: Helm support for KubernetesContainerFactory (#209)

This is an automated email from the ASF dual-hosted git repository.

csantanapr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git


The following commit(s) were added to refs/heads/master by this push:
     new 78f8381  Helm support for KubernetesContainerFactory (#209)
78f8381 is described below

commit 78f83818139e2fc59cda811af4188177a40c144b
Author: David Grove <dg...@users.noreply.github.com>
AuthorDate: Mon Jun 4 10:08:55 2018 -0400

    Helm support for KubernetesContainerFactory (#209)
    
    Add support for KubernetesContainerFactory both with and
    without the invokerAgent. Also some minor cleanup of
    invoker.yaml to better factor out common pieces.
    
    Flip Travis testing to be primarily helm-based.
    
    Both the KubernetesContainerFactory and akka cluster discovery
    via the kubernetes API (used when deploying multiple controllers)
    requite RBAC authorization mode. Add flag to enable RBAC to
    the `minikube start` commands and drop minikube 0.23.0/kube 1.7.4
    (which do not support this mode).
---
 .travis.yml                        |   9 ++-
 README.md                          |   1 -
 docs/setting_up_minikube/README.md |  12 ++--
 helm/templates/_helpersInvoker.tpl |  39 ++++++++++++
 helm/templates/invoker.yaml        | 121 ++++++++++++++++++++++++++-----------
 helm/templates/rolebindings.yaml   |  45 ++++++++++++++
 helm/values.yaml                   |  12 ++++
 tools/travis/build-helm.sh         |   7 +++
 tools/travis/build.sh              |   2 +-
 tools/travis/setup.sh              |   2 +-
 10 files changed, 200 insertions(+), 50 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 13bb8a6..888237c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,11 +7,10 @@ env:
   - secure: d7CuMXbhT83W2x78qiLwgogX1+3aPicd1PlTwwNNDN6QSkImbxareyKThnsqlHIiNj3o5l5DBuiYjy7wrF/xD1g8BQMmTwm99DRx5q3CI3Im3VCi/ZK8SaNjuOy24d7cf5k2tB/87Gk7zmKsMDYm+fpCl+GpgUmIEeIwthiAxuXSDWZ8eQPIptmxj56DeFRNouvXG+dEUtBfWiwN27UPxNKExCixFnegmdtffLbz6hhst7BHr5Ry9acbycre98PCwWZcu9lxFs+SJ1kvnzX2iue4otmDkF1WkJjxaOFPJVs/D3YItg+neLCSxjwBskPed+Fct8bOjcM/uVROJPNIq5icBmaPX2isH0lvtxOeVw/dmioWYXYPN9ygBOe4eO/vtPllN0bcAUo5xl9jXev8ciAozYrYpHVh9Fplfd81rcYTeYzALmRJBdoiWoc3KQGzwGc9sB1ffmy+KWgG9T0zbnS4fALSR4PS [...]
   - secure: CJtnU94HTDqd4A6uvhFl8IpnmU+wTdlzb8bPBFUl/lI/VKXiRrYpgJdKUro5xEoxFKuqMprLhbyf66niyWLTIeogjUAEu/h/o2dBVeGgSGKoqC0hQgqvnxKFeGlzFJ0XuEs3vbStJGRnQszGsfnnDrscJtR0x9X+1w4aBKI7iPyyuFtVkDD1UsmBbSi+M8FTeq7G7A0reMDaey7uog3CFCpIMl4geshcohQEcKEGbnXQZoLPFpb7cBOE83VXBJ7Y7Dgf/U4keiLovvnuJThGKZm/SVV2KlELmBmtmbx3rMT6Vb5k9ChSdRWapromNnnzmJBIQ5Scc2mwV3A93/SMha1F3IlYpDKs5djfTw8jZfVnuiou7HhTaRjHkmmcwP12/k30gLe2kw0Vezg1TCY4zgtOpcmCxc8RHEy0ceA74rKvRi8LbexTCwX+iAMQFn/pSrh/OqAq/50JbLyczcoO1zXWS38txUQN [...]
   matrix:
-    - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.7.4 TRAVIS_MINIKUBE_VERSION=v0.23.0
-    - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
-    - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
-    - TRAVIS_USE_HELM=true  OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
-    - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=kube   TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+    - TRAVIS_USE_HELM=true  OW_CONTAINER_FACTORY=docker     TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+    - TRAVIS_USE_HELM=true  OW_CONTAINER_FACTORY=docker     TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+    - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker     TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+    - TRAVIS_USE_HELM=true  OW_CONTAINER_FACTORY=kubernetes TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
 
 services:
   - docker
diff --git a/README.md b/README.md
index 36c6be8..3657811 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,6 @@ Travis CI testing.
 
 | Kubernetes Version | Minikube Version |
 --- | --- |
-1.7.4 | 0.23.0 |
 1.8.0 | 0.25.2 |
 1.9.0 | 0.25.2 |
 
diff --git a/docs/setting_up_minikube/README.md b/docs/setting_up_minikube/README.md
index 71a45b5..00779e3 100644
--- a/docs/setting_up_minikube/README.md
+++ b/docs/setting_up_minikube/README.md
@@ -40,12 +40,12 @@ asdf plugin-add kubectl
 asdf plugin-add minikube
 ```
 
-### Install asdf plugin minikube@0.23.0 and kubectl@1.7.4
+### Install asdf plugin minikube@0.25.2 and kubectl@1.9.0
 ```
-asdf install kubectl 1.7.4
-asdf global kubectl 1.7.4
-asdf install minikube 0.23.0
-asdf global minikube 0.23.0
+asdf install kubectl 1.9.0
+asdf global kubectl 1.9.0
+asdf install minikube 0.25.2
+asdf global minikube 0.25.2
 ```
 
 ## Create the minikube VM
@@ -54,7 +54,7 @@ If you have a larger machine, you may want to provision more (especially more me
 
 Start Minikube with:
 ```
-minikube start --cpus 2 --memory 4096 --kubernetes-version=v1.7.4
+minikube start --cpus 2 --memory 4096 --kubernetes-version=v1.9.0 --extra-config=apiserver.Authorization.Mode=RBAC
 ```
 
 ## Setup Docker network in promiscuous mode
diff --git a/helm/templates/_helpersInvoker.tpl b/helm/templates/_helpersInvoker.tpl
new file mode 100644
index 0000000..4b14cc7
--- /dev/null
+++ b/helm/templates/_helpersInvoker.tpl
@@ -0,0 +1,39 @@
+{{- define "docker_volumes" -}}
+- name: cgroup
+  hostPath:
+    path: "/sys/fs/cgroup"
+- name: runc
+  hostPath:
+    path: "/run/runc"
+- name: dockerrootdir
+  hostPath:
+    path: "/var/lib/docker/containers"
+- name: dockersock
+  hostPath:
+    path: "/var/run/docker.sock"
+{{- end -}}
+
+{{- define "docker_volume_mounts" -}}
+- name: cgroup
+  mountPath: "/sys/fs/cgroup"
+- name: runc
+  mountPath: "/run/runc"
+- name: dockersock
+  mountPath: "/var/run/docker.sock"
+- name: dockerrootdir
+  mountPath: "/containers"
+{{- end -}}
+
+{{- define "docker_pull_runtimes" -}}
+- name: docker-pull-runtimes
+  imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
+  image: {{ .Values.invoker.pullRuntimesImage | quote }}
+  volumeMounts:
+  - name: dockersock
+    mountPath: "/var/run/docker.sock"
+  env:
+    # action runtimes
+    - name: "RUNTIMES_MANIFEST"
+      value: {{ template "runtimes_manifest" . }}
+{{- end -}}
+
diff --git a/helm/templates/invoker.yaml b/helm/templates/invoker.yaml
index d5ac64e..d1c897f 100644
--- a/helm/templates/invoker.yaml
+++ b/helm/templates/invoker.yaml
@@ -1,53 +1,52 @@
 # Licensed to the Apache Software Foundation (ASF) under one or more contributor
 # license agreements; and to You under the Apache License, Version 2.0.
 
+---
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
 apiVersion: extensions/v1beta1
 kind: DaemonSet
+{{- else if eq .Values.invoker.containerFactory.impl "kubernetes" -}}
+apiVersion: apps/v1beta1
+kind: StatefulSet
+{{- end }}
 metadata:
   name: {{ .Values.invoker.name | quote }}
   namespace: {{ .Release.Namespace | quote }}
   labels:
     name: {{ .Values.invoker.name | quote }}
 spec:
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+  replicas: {{ .Values.invoker.containerFactory.kubernetes.replicaCount }}
+{{- end }}
   template:
     metadata:
       labels:
         name: {{ .Values.invoker.name | quote }}
     spec:
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+      serviceAccountName: {{ .Values.invoker.name | quote }}
+{{- end }}
       restartPolicy: {{ .Values.invoker.restartPolicy }}
 
       affinity:
 {{ include "affinity.invoker" . | indent 8 }}
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+{{ include "affinity.selfAntiAffinity" ( .Values.invoker.name | quote ) | indent 8 }}
+{{- end }}
 
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
       volumes:
-      - name: cgroup
-        hostPath:
-          path: "/sys/fs/cgroup"
-      - name: runc
-        hostPath:
-          path: "/run/runc"
-      - name: dockerrootdir
-        hostPath:
-          path: "/var/lib/docker/containers"
-      - name: dockersock
-        hostPath:
-          path: "/var/run/docker.sock"
+{{ include "docker_volumes" . | indent 6 }}
+{{- end }}
 
       initContainers:
       # Wait for a controller to be up (which implies kafka, zookeeper, couchdb are all up as well).
 {{ include "readiness.waitForController" . | indent 6 }}
 
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
       # Pull images for all default runtimes before starting invoker
-      - name: docker-pull-runtimes
-        imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
-        image: openwhisk/kube-docker-pull
-        volumeMounts:
-        - name: dockersock
-          mountPath: "/var/run/docker.sock"
-        env:
-          # action runtimes
-          - name: "RUNTIMES_MANIFEST"
-            value: {{ template "runtimes_manifest" . }}
+{{ include "docker_pull_runtimes" . | indent 6 }}
+{{- end }}
 
       containers:
       - name: {{ .Values.invoker.name | quote }}
@@ -56,7 +55,7 @@ spec:
         command: [ "/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /init.sh"]
         env:
           - name: "PORT"
-            value: "8080"
+            value: {{ .Values.invoker.port | quote }}
           - name: "WHISK_API_HOST_NAME"
             valueFrom:
               configMapKeyRef:
@@ -75,19 +74,26 @@ spec:
           - name: "DOCKER_REGISTRY"
             value: ""
 
-          # Invoker name is name of the Kube node when using DaemonSet
+          # Invoker name is the name of the node (DaemonSet) or pod (StatefulSet)
           - name: "INVOKER_NAME"
             valueFrom:
               fieldRef:
-                fieldPath: spec.nodeName
+                fieldPath: {{ if eq .Values.invoker.containerFactory.impl "docker" }} spec.nodeName {{ else }} metadata.name {{ end }}
 
           # Java options
           - name: "JAVA_OPTS"
-            value: "-Xmx2g"
+            value: {{ .Values.invoker.jvmOptions | quote }}
 
           # Invoker options
           - name: "INVOKER_OPTS"
-            value: {{ .Values.invoker.options | quote }}
+            value: "{{ .Values.invoker.options }} {{ if eq .Values.invoker.containerFactory.impl "docker" }} -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.docker.DockerContainerFactoryProvider {{ else }} -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider {{ end }}"
+
+{{ if .Values.invoker.containerFactory.kubernetes.agent.enabled }}
+          - name: "CONFIG_whisk_kubernetes_invokerAgent_enabled"
+            value:  "TRUE"
+          - name: "CONFIG_whisk_kubernetes_invokerAgent_port"
+            value: {{ .Values.invoker.containerFactory.kubernetes.agent.port | quote }}
+{{ end }}
 
           # action runtimes
           - name: "RUNTIMES_MANIFEST"
@@ -114,13 +120,56 @@ spec:
 
         ports:
         - name: invoker
-          containerPort: 8080
+          containerPort: {{ .Values.invoker.port }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
+        volumeMounts:
+{{ include "docker_volume_mounts" . | indent 8 }}
+{{- end }}
+
+{{ if .Values.invoker.containerFactory.kubernetes.agent.enabled }}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+  labels:
+    name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+spec:
+  template:
+    metadata:
+      labels:
+        name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+    spec:
+      restartPolicy: Always
+      hostNetwork: true
+
+      # TODO: disabled affinity until user-action pods are
+      #       created by KubernetesContainerFacotry with the
+      #       same affinity rules.
+      #       Requires extension to upstream kube java client
+      #       run only on nodes labeled with openwhisk-role=invoker
+
+      volumes:
+{{ include "docker_volumes" . | indent 6 }}
+      - name: userlogs
+        emptyDir: {}
+
+      initContainers:
+      # Pull images for all default runtimes before starting invoker
+{{ include "docker_pull_runtimes" . | indent 6 }}
+
+      containers:
+      - name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+        imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
+        image: {{ .Values.invoker.containerFactory.kubernetes.agent.image }}
+        securityContext:
+          privileged: true
+        ports:
+        - name: agent
+          containerPort: {{ .Values.invoker.containerFactory.kubernetes.agent.port }}
         volumeMounts:
-        - name: cgroup
-          mountPath: "/sys/fs/cgroup"
-        - name: runc
-          mountPath: "/run/runc"
-        - name: dockersock
-          mountPath: "/var/run/docker.sock"
-        - name: dockerrootdir
-          mountPath: "/containers"
+{{ include "docker_volume_mounts" . | indent 8 }}
+        - name: userlogs
+          mountPath: "/action-logs"
+{{- end }}
diff --git a/helm/templates/rolebindings.yaml b/helm/templates/rolebindings.yaml
index 298bc5b..b8b4bca 100644
--- a/helm/templates/rolebindings.yaml
+++ b/helm/templates/rolebindings.yaml
@@ -22,3 +22,48 @@ subjects:
   - kind: ServiceAccount
     name: ow-core
     namespace: {{ .Release.Namespace | quote }}
+
+
+{{ if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+# When using KubernetesContainerFactory, invoker pods need extensive
+# permissions to manage pods and deployments. The ability to create
+# pods can enable privilege escalation attacks, so restrict it to a
+# ServiceAccount that is only used for the invokers and only defined
+# when using KubernetesContainerFactory.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: {{ .Release.Namespace | quote }}
+  name: {{ .Values.invoker.name | quote }}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  namespace: {{ .Release.Namespace | quote }}
+  name: {{ .Values.invoker.name | quote }}
+rules:
+- apiGroups: ["extensions"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+  resources: ["pods/log"]
+  verbs: ["get", "list"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+  namespace: {{ .Release.Namespace | quote }}
+  name: {{ .Values.invoker.name | quote }}
+subjects:
+- kind: ServiceAccount
+  namespace: {{ .Release.Namespace | quote }}
+  name: {{ .Values.invoker.name | quote }}
+roleRef:
+  kind: Role
+  name: {{ .Values.invoker.name | quote }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
index d74d452..5944d73 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -94,7 +94,19 @@ invoker:
   image: "openwhisk/invoker"
   imagePullPolicy: "IfNotPresent"
   restartPolicy: "Always"
+  port: 8080
+  pullRuntimesImage: "openwhisk/kube-docker-pull"
   options: ""
+  jvmOptions: "-Xmx512M"
+  containerFactory:
+    impl: "docker"
+    kubernetes:
+      replicaCount: 1
+      agent:
+        name: "invoker-agent"
+        image: "openwhisk/kube-invoker-agent"
+        enabled: false
+        port: 3233
 
 # API Gateway configurations
 apigw:
diff --git a/tools/travis/build-helm.sh b/tools/travis/build-helm.sh
index bb3ce6b..cdb39a5 100755
--- a/tools/travis/build-helm.sh
+++ b/tools/travis/build-helm.sh
@@ -154,6 +154,13 @@ whisk:
     api_host: $WSK_HOST:$WSK_PORT
   runtimes: "runtimes-minimal-travis.json"
 
+invoker:
+  containerFactory:
+    impl: $OW_CONTAINER_FACTORY
+    kubernetes:
+      agent:
+        enabled: true
+
 nginx:
   httpsNodePort: $WSK_PORT
 EOF
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index 626b1d2..48f1348 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -234,7 +234,7 @@ pushd kubernetes/invoker
         echo "Deploying invoker using DockerContainerFactory"
         kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-dcf.env
         kubectl apply -f invoker-dcf.yml
-    elif [ "$OW_CONTAINER_FACTORY" = "kube" ]; then
+    elif [ "$OW_CONTAINER_FACTORY" = "kubernetes" ]; then
         echo "Deploying invoker using KubernetesContainerFactory"
         kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-k8scf.env
         kubectl apply -f invoker-agent.yml
diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh
index 251fde8..7ecc8a6 100755
--- a/tools/travis/setup.sh
+++ b/tools/travis/setup.sh
@@ -59,7 +59,7 @@ mkdir $HOME/.kube || true
 touch $HOME/.kube/config
 
 export KUBECONFIG=$HOME/.kube/config
-sudo -E /usr/local/bin/minikube start --vm-driver=none --kubernetes-version=$TRAVIS_KUBE_VERSION
+sudo -E /usr/local/bin/minikube start --vm-driver=none --extra-config=apiserver.Authorization.Mode=RBAC --kubernetes-version=$TRAVIS_KUBE_VERSION
 
 # Wait until we have a ready node in minikube
 TIMEOUT=0

-- 
To stop receiving notification emails like this one, please contact
csantanapr@apache.org.