You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by cs...@apache.org on 2018/06/04 14:09:05 UTC
[incubator-openwhisk-deploy-kube] branch master updated: Helm
support for KubernetesContainerFactory (#209)
This is an automated email from the ASF dual-hosted git repository.
csantanapr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git
The following commit(s) were added to refs/heads/master by this push:
new 78f8381 Helm support for KubernetesContainerFactory (#209)
78f8381 is described below
commit 78f83818139e2fc59cda811af4188177a40c144b
Author: David Grove <dg...@users.noreply.github.com>
AuthorDate: Mon Jun 4 10:08:55 2018 -0400
Helm support for KubernetesContainerFactory (#209)
Add support for KubernetesContainerFactory both with and
without the invokerAgent. Also some minor cleanup of
invoker.yaml to better factor out common pieces.
Flip Travis testing to be primarily helm-based.
Both the KubernetesContainerFactory and akka cluster discovery
via the kubernetes API (used when deploying multiple controllers)
requite RBAC authorization mode. Add flag to enable RBAC to
the `minikube start` commands and drop minikube 0.23.0/kube 1.7.4
(which do not support this mode).
---
.travis.yml | 9 ++-
README.md | 1 -
docs/setting_up_minikube/README.md | 12 ++--
helm/templates/_helpersInvoker.tpl | 39 ++++++++++++
helm/templates/invoker.yaml | 121 ++++++++++++++++++++++++++-----------
helm/templates/rolebindings.yaml | 45 ++++++++++++++
helm/values.yaml | 12 ++++
tools/travis/build-helm.sh | 7 +++
tools/travis/build.sh | 2 +-
tools/travis/setup.sh | 2 +-
10 files changed, 200 insertions(+), 50 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 13bb8a6..888237c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,11 +7,10 @@ env:
- secure: d7CuMXbhT83W2x78qiLwgogX1+3aPicd1PlTwwNNDN6QSkImbxareyKThnsqlHIiNj3o5l5DBuiYjy7wrF/xD1g8BQMmTwm99DRx5q3CI3Im3VCi/ZK8SaNjuOy24d7cf5k2tB/87Gk7zmKsMDYm+fpCl+GpgUmIEeIwthiAxuXSDWZ8eQPIptmxj56DeFRNouvXG+dEUtBfWiwN27UPxNKExCixFnegmdtffLbz6hhst7BHr5Ry9acbycre98PCwWZcu9lxFs+SJ1kvnzX2iue4otmDkF1WkJjxaOFPJVs/D3YItg+neLCSxjwBskPed+Fct8bOjcM/uVROJPNIq5icBmaPX2isH0lvtxOeVw/dmioWYXYPN9ygBOe4eO/vtPllN0bcAUo5xl9jXev8ciAozYrYpHVh9Fplfd81rcYTeYzALmRJBdoiWoc3KQGzwGc9sB1ffmy+KWgG9T0zbnS4fALSR4PS [...]
- secure: CJtnU94HTDqd4A6uvhFl8IpnmU+wTdlzb8bPBFUl/lI/VKXiRrYpgJdKUro5xEoxFKuqMprLhbyf66niyWLTIeogjUAEu/h/o2dBVeGgSGKoqC0hQgqvnxKFeGlzFJ0XuEs3vbStJGRnQszGsfnnDrscJtR0x9X+1w4aBKI7iPyyuFtVkDD1UsmBbSi+M8FTeq7G7A0reMDaey7uog3CFCpIMl4geshcohQEcKEGbnXQZoLPFpb7cBOE83VXBJ7Y7Dgf/U4keiLovvnuJThGKZm/SVV2KlELmBmtmbx3rMT6Vb5k9ChSdRWapromNnnzmJBIQ5Scc2mwV3A93/SMha1F3IlYpDKs5djfTw8jZfVnuiou7HhTaRjHkmmcwP12/k30gLe2kw0Vezg1TCY4zgtOpcmCxc8RHEy0ceA74rKvRi8LbexTCwX+iAMQFn/pSrh/OqAq/50JbLyczcoO1zXWS38txUQN [...]
matrix:
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.7.4 TRAVIS_MINIKUBE_VERSION=v0.23.0
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
- - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=kube TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=kubernetes TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
services:
- docker
diff --git a/README.md b/README.md
index 36c6be8..3657811 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,6 @@ Travis CI testing.
| Kubernetes Version | Minikube Version |
--- | --- |
-1.7.4 | 0.23.0 |
1.8.0 | 0.25.2 |
1.9.0 | 0.25.2 |
diff --git a/docs/setting_up_minikube/README.md b/docs/setting_up_minikube/README.md
index 71a45b5..00779e3 100644
--- a/docs/setting_up_minikube/README.md
+++ b/docs/setting_up_minikube/README.md
@@ -40,12 +40,12 @@ asdf plugin-add kubectl
asdf plugin-add minikube
```
-### Install asdf plugin minikube@0.23.0 and kubectl@1.7.4
+### Install asdf plugin minikube@0.25.2 and kubectl@1.9.0
```
-asdf install kubectl 1.7.4
-asdf global kubectl 1.7.4
-asdf install minikube 0.23.0
-asdf global minikube 0.23.0
+asdf install kubectl 1.9.0
+asdf global kubectl 1.9.0
+asdf install minikube 0.25.2
+asdf global minikube 0.25.2
```
## Create the minikube VM
@@ -54,7 +54,7 @@ If you have a larger machine, you may want to provision more (especially more me
Start Minikube with:
```
-minikube start --cpus 2 --memory 4096 --kubernetes-version=v1.7.4
+minikube start --cpus 2 --memory 4096 --kubernetes-version=v1.9.0 --extra-config=apiserver.Authorization.Mode=RBAC
```
## Setup Docker network in promiscuous mode
diff --git a/helm/templates/_helpersInvoker.tpl b/helm/templates/_helpersInvoker.tpl
new file mode 100644
index 0000000..4b14cc7
--- /dev/null
+++ b/helm/templates/_helpersInvoker.tpl
@@ -0,0 +1,39 @@
+{{- define "docker_volumes" -}}
+- name: cgroup
+ hostPath:
+ path: "/sys/fs/cgroup"
+- name: runc
+ hostPath:
+ path: "/run/runc"
+- name: dockerrootdir
+ hostPath:
+ path: "/var/lib/docker/containers"
+- name: dockersock
+ hostPath:
+ path: "/var/run/docker.sock"
+{{- end -}}
+
+{{- define "docker_volume_mounts" -}}
+- name: cgroup
+ mountPath: "/sys/fs/cgroup"
+- name: runc
+ mountPath: "/run/runc"
+- name: dockersock
+ mountPath: "/var/run/docker.sock"
+- name: dockerrootdir
+ mountPath: "/containers"
+{{- end -}}
+
+{{- define "docker_pull_runtimes" -}}
+- name: docker-pull-runtimes
+ imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
+ image: {{ .Values.invoker.pullRuntimesImage | quote }}
+ volumeMounts:
+ - name: dockersock
+ mountPath: "/var/run/docker.sock"
+ env:
+ # action runtimes
+ - name: "RUNTIMES_MANIFEST"
+ value: {{ template "runtimes_manifest" . }}
+{{- end -}}
+
diff --git a/helm/templates/invoker.yaml b/helm/templates/invoker.yaml
index d5ac64e..d1c897f 100644
--- a/helm/templates/invoker.yaml
+++ b/helm/templates/invoker.yaml
@@ -1,53 +1,52 @@
# Licensed to the Apache Software Foundation (ASF) under one or more contributor
# license agreements; and to You under the Apache License, Version 2.0.
+---
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
apiVersion: extensions/v1beta1
kind: DaemonSet
+{{- else if eq .Values.invoker.containerFactory.impl "kubernetes" -}}
+apiVersion: apps/v1beta1
+kind: StatefulSet
+{{- end }}
metadata:
name: {{ .Values.invoker.name | quote }}
namespace: {{ .Release.Namespace | quote }}
labels:
name: {{ .Values.invoker.name | quote }}
spec:
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+ replicas: {{ .Values.invoker.containerFactory.kubernetes.replicaCount }}
+{{- end }}
template:
metadata:
labels:
name: {{ .Values.invoker.name | quote }}
spec:
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+ serviceAccountName: {{ .Values.invoker.name | quote }}
+{{- end }}
restartPolicy: {{ .Values.invoker.restartPolicy }}
affinity:
{{ include "affinity.invoker" . | indent 8 }}
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+{{ include "affinity.selfAntiAffinity" ( .Values.invoker.name | quote ) | indent 8 }}
+{{- end }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
volumes:
- - name: cgroup
- hostPath:
- path: "/sys/fs/cgroup"
- - name: runc
- hostPath:
- path: "/run/runc"
- - name: dockerrootdir
- hostPath:
- path: "/var/lib/docker/containers"
- - name: dockersock
- hostPath:
- path: "/var/run/docker.sock"
+{{ include "docker_volumes" . | indent 6 }}
+{{- end }}
initContainers:
# Wait for a controller to be up (which implies kafka, zookeeper, couchdb are all up as well).
{{ include "readiness.waitForController" . | indent 6 }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
# Pull images for all default runtimes before starting invoker
- - name: docker-pull-runtimes
- imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
- image: openwhisk/kube-docker-pull
- volumeMounts:
- - name: dockersock
- mountPath: "/var/run/docker.sock"
- env:
- # action runtimes
- - name: "RUNTIMES_MANIFEST"
- value: {{ template "runtimes_manifest" . }}
+{{ include "docker_pull_runtimes" . | indent 6 }}
+{{- end }}
containers:
- name: {{ .Values.invoker.name | quote }}
@@ -56,7 +55,7 @@ spec:
command: [ "/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /init.sh"]
env:
- name: "PORT"
- value: "8080"
+ value: {{ .Values.invoker.port | quote }}
- name: "WHISK_API_HOST_NAME"
valueFrom:
configMapKeyRef:
@@ -75,19 +74,26 @@ spec:
- name: "DOCKER_REGISTRY"
value: ""
- # Invoker name is name of the Kube node when using DaemonSet
+ # Invoker name is the name of the node (DaemonSet) or pod (StatefulSet)
- name: "INVOKER_NAME"
valueFrom:
fieldRef:
- fieldPath: spec.nodeName
+ fieldPath: {{ if eq .Values.invoker.containerFactory.impl "docker" }} spec.nodeName {{ else }} metadata.name {{ end }}
# Java options
- name: "JAVA_OPTS"
- value: "-Xmx2g"
+ value: {{ .Values.invoker.jvmOptions | quote }}
# Invoker options
- name: "INVOKER_OPTS"
- value: {{ .Values.invoker.options | quote }}
+ value: "{{ .Values.invoker.options }} {{ if eq .Values.invoker.containerFactory.impl "docker" }} -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.docker.DockerContainerFactoryProvider {{ else }} -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider {{ end }}"
+
+{{ if .Values.invoker.containerFactory.kubernetes.agent.enabled }}
+ - name: "CONFIG_whisk_kubernetes_invokerAgent_enabled"
+ value: "TRUE"
+ - name: "CONFIG_whisk_kubernetes_invokerAgent_port"
+ value: {{ .Values.invoker.containerFactory.kubernetes.agent.port | quote }}
+{{ end }}
# action runtimes
- name: "RUNTIMES_MANIFEST"
@@ -114,13 +120,56 @@ spec:
ports:
- name: invoker
- containerPort: 8080
+ containerPort: {{ .Values.invoker.port }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
+ volumeMounts:
+{{ include "docker_volume_mounts" . | indent 8 }}
+{{- end }}
+
+{{ if .Values.invoker.containerFactory.kubernetes.agent.enabled }}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+spec:
+ template:
+ metadata:
+ labels:
+ name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+ spec:
+ restartPolicy: Always
+ hostNetwork: true
+
+ # TODO: disabled affinity until user-action pods are
+ # created by KubernetesContainerFacotry with the
+ # same affinity rules.
+ # Requires extension to upstream kube java client
+ # run only on nodes labeled with openwhisk-role=invoker
+
+ volumes:
+{{ include "docker_volumes" . | indent 6 }}
+ - name: userlogs
+ emptyDir: {}
+
+ initContainers:
+ # Pull images for all default runtimes before starting invoker
+{{ include "docker_pull_runtimes" . | indent 6 }}
+
+ containers:
+ - name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+ imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
+ image: {{ .Values.invoker.containerFactory.kubernetes.agent.image }}
+ securityContext:
+ privileged: true
+ ports:
+ - name: agent
+ containerPort: {{ .Values.invoker.containerFactory.kubernetes.agent.port }}
volumeMounts:
- - name: cgroup
- mountPath: "/sys/fs/cgroup"
- - name: runc
- mountPath: "/run/runc"
- - name: dockersock
- mountPath: "/var/run/docker.sock"
- - name: dockerrootdir
- mountPath: "/containers"
+{{ include "docker_volume_mounts" . | indent 8 }}
+ - name: userlogs
+ mountPath: "/action-logs"
+{{- end }}
diff --git a/helm/templates/rolebindings.yaml b/helm/templates/rolebindings.yaml
index 298bc5b..b8b4bca 100644
--- a/helm/templates/rolebindings.yaml
+++ b/helm/templates/rolebindings.yaml
@@ -22,3 +22,48 @@ subjects:
- kind: ServiceAccount
name: ow-core
namespace: {{ .Release.Namespace | quote }}
+
+
+{{ if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+# When using KubernetesContainerFactory, invoker pods need extensive
+# permissions to manage pods and deployments. The ability to create
+# pods can enable privilege escalation attacks, so restrict it to a
+# ServiceAccount that is only used for the invokers and only defined
+# when using KubernetesContainerFactory.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+rules:
+- apiGroups: ["extensions"]
+ resources: ["deployments"]
+ verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+ resources: ["pods/log"]
+ verbs: ["get", "list"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+subjects:
+- kind: ServiceAccount
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+roleRef:
+ kind: Role
+ name: {{ .Values.invoker.name | quote }}
+ apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
index d74d452..5944d73 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -94,7 +94,19 @@ invoker:
image: "openwhisk/invoker"
imagePullPolicy: "IfNotPresent"
restartPolicy: "Always"
+ port: 8080
+ pullRuntimesImage: "openwhisk/kube-docker-pull"
options: ""
+ jvmOptions: "-Xmx512M"
+ containerFactory:
+ impl: "docker"
+ kubernetes:
+ replicaCount: 1
+ agent:
+ name: "invoker-agent"
+ image: "openwhisk/kube-invoker-agent"
+ enabled: false
+ port: 3233
# API Gateway configurations
apigw:
diff --git a/tools/travis/build-helm.sh b/tools/travis/build-helm.sh
index bb3ce6b..cdb39a5 100755
--- a/tools/travis/build-helm.sh
+++ b/tools/travis/build-helm.sh
@@ -154,6 +154,13 @@ whisk:
api_host: $WSK_HOST:$WSK_PORT
runtimes: "runtimes-minimal-travis.json"
+invoker:
+ containerFactory:
+ impl: $OW_CONTAINER_FACTORY
+ kubernetes:
+ agent:
+ enabled: true
+
nginx:
httpsNodePort: $WSK_PORT
EOF
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index 626b1d2..48f1348 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -234,7 +234,7 @@ pushd kubernetes/invoker
echo "Deploying invoker using DockerContainerFactory"
kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-dcf.env
kubectl apply -f invoker-dcf.yml
- elif [ "$OW_CONTAINER_FACTORY" = "kube" ]; then
+ elif [ "$OW_CONTAINER_FACTORY" = "kubernetes" ]; then
echo "Deploying invoker using KubernetesContainerFactory"
kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-k8scf.env
kubectl apply -f invoker-agent.yml
diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh
index 251fde8..7ecc8a6 100755
--- a/tools/travis/setup.sh
+++ b/tools/travis/setup.sh
@@ -59,7 +59,7 @@ mkdir $HOME/.kube || true
touch $HOME/.kube/config
export KUBECONFIG=$HOME/.kube/config
-sudo -E /usr/local/bin/minikube start --vm-driver=none --kubernetes-version=$TRAVIS_KUBE_VERSION
+sudo -E /usr/local/bin/minikube start --vm-driver=none --extra-config=apiserver.Authorization.Mode=RBAC --kubernetes-version=$TRAVIS_KUBE_VERSION
# Wait until we have a ready node in minikube
TIMEOUT=0
--
To stop receiving notification emails like this one, please contact
csantanapr@apache.org.