You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/06/04 14:08:57 UTC
[GitHub] csantanapr closed pull request #209: helm support for
KubernetesContainerFactory
csantanapr closed pull request #209: helm support for KubernetesContainerFactory
URL: https://github.com/apache/incubator-openwhisk-deploy-kube/pull/209
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.travis.yml b/.travis.yml
index 13bb8a6..888237c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -7,11 +7,10 @@ env:
- secure: d7CuMXbhT83W2x78qiLwgogX1+3aPicd1PlTwwNNDN6QSkImbxareyKThnsqlHIiNj3o5l5DBuiYjy7wrF/xD1g8BQMmTwm99DRx5q3CI3Im3VCi/ZK8SaNjuOy24d7cf5k2tB/87Gk7zmKsMDYm+fpCl+GpgUmIEeIwthiAxuXSDWZ8eQPIptmxj56DeFRNouvXG+dEUtBfWiwN27UPxNKExCixFnegmdtffLbz6hhst7BHr5Ry9acbycre98PCwWZcu9lxFs+SJ1kvnzX2iue4otmDkF1WkJjxaOFPJVs/D3YItg+neLCSxjwBskPed+Fct8bOjcM/uVROJPNIq5icBmaPX2isH0lvtxOeVw/dmioWYXYPN9ygBOe4eO/vtPllN0bcAUo5xl9jXev8ciAozYrYpHVh9Fplfd81rcYTeYzALmRJBdoiWoc3KQGzwGc9sB1ffmy+KWgG9T0zbnS4fALSR4PSzyNlKSLXw9vuvdNP0OBYtO+6yTJXavIxqmDoj64Lb93n+uGPatnaIGPTKEEBMJTSjsgYVEfxzzZuxUT9Ldkf2lzqvN2PCllGoMqxkgsdb8i4v4QgRaMWBDbKa5Va4k0O4dnhRmtdbJavOSKN6fECJbyfoJlV1VvJGxk5znVLRznBmUPBKbNccyPZJULugKD3QIh4q8Q5jBU=
- secure: CJtnU94HTDqd4A6uvhFl8IpnmU+wTdlzb8bPBFUl/lI/VKXiRrYpgJdKUro5xEoxFKuqMprLhbyf66niyWLTIeogjUAEu/h/o2dBVeGgSGKoqC0hQgqvnxKFeGlzFJ0XuEs3vbStJGRnQszGsfnnDrscJtR0x9X+1w4aBKI7iPyyuFtVkDD1UsmBbSi+M8FTeq7G7A0reMDaey7uog3CFCpIMl4geshcohQEcKEGbnXQZoLPFpb7cBOE83VXBJ7Y7Dgf/U4keiLovvnuJThGKZm/SVV2KlELmBmtmbx3rMT6Vb5k9ChSdRWapromNnnzmJBIQ5Scc2mwV3A93/SMha1F3IlYpDKs5djfTw8jZfVnuiou7HhTaRjHkmmcwP12/k30gLe2kw0Vezg1TCY4zgtOpcmCxc8RHEy0ceA74rKvRi8LbexTCwX+iAMQFn/pSrh/OqAq/50JbLyczcoO1zXWS38txUQNLW8i+XllhCg9pjkjyfBeGjOOcWiVIz9rWJd2XufjSXDcj6xoZHtkh1XDt1CnVkpsYKtyyZucQnhUM9ebmaWqbSW2+bpqC/2hI+G+kOyyCesGdB1q+VmN1augMMs6RgWjk4yw5dyLAshATSoUlE8KH2cDcJL19r4ECaQ99PSLwxoB89yfPoJiNc42vwxRdsLmB1BMNyPa81Y=
matrix:
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.7.4 TRAVIS_MINIKUBE_VERSION=v0.23.0
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
- - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
- - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=kube TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.8.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=false OW_CONTAINER_FACTORY=docker TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
+ - TRAVIS_USE_HELM=true OW_CONTAINER_FACTORY=kubernetes TRAVIS_KUBE_VERSION=v1.9.0 TRAVIS_MINIKUBE_VERSION=v0.25.2
services:
- docker
diff --git a/README.md b/README.md
index 36c6be8..3657811 100644
--- a/README.md
+++ b/README.md
@@ -53,7 +53,6 @@ Travis CI testing.
| Kubernetes Version | Minikube Version |
--- | --- |
-1.7.4 | 0.23.0 |
1.8.0 | 0.25.2 |
1.9.0 | 0.25.2 |
diff --git a/docs/setting_up_minikube/README.md b/docs/setting_up_minikube/README.md
index 71a45b5..00779e3 100644
--- a/docs/setting_up_minikube/README.md
+++ b/docs/setting_up_minikube/README.md
@@ -40,12 +40,12 @@ asdf plugin-add kubectl
asdf plugin-add minikube
```
-### Install asdf plugin minikube@0.23.0 and kubectl@1.7.4
+### Install asdf plugin minikube@0.25.2 and kubectl@1.9.0
```
-asdf install kubectl 1.7.4
-asdf global kubectl 1.7.4
-asdf install minikube 0.23.0
-asdf global minikube 0.23.0
+asdf install kubectl 1.9.0
+asdf global kubectl 1.9.0
+asdf install minikube 0.25.2
+asdf global minikube 0.25.2
```
## Create the minikube VM
@@ -54,7 +54,7 @@ If you have a larger machine, you may want to provision more (especially more me
Start Minikube with:
```
-minikube start --cpus 2 --memory 4096 --kubernetes-version=v1.7.4
+minikube start --cpus 2 --memory 4096 --kubernetes-version=v1.9.0 --extra-config=apiserver.Authorization.Mode=RBAC
```
## Setup Docker network in promiscuous mode
diff --git a/helm/templates/_helpersInvoker.tpl b/helm/templates/_helpersInvoker.tpl
new file mode 100644
index 0000000..4b14cc7
--- /dev/null
+++ b/helm/templates/_helpersInvoker.tpl
@@ -0,0 +1,39 @@
+{{- define "docker_volumes" -}}
+- name: cgroup
+ hostPath:
+ path: "/sys/fs/cgroup"
+- name: runc
+ hostPath:
+ path: "/run/runc"
+- name: dockerrootdir
+ hostPath:
+ path: "/var/lib/docker/containers"
+- name: dockersock
+ hostPath:
+ path: "/var/run/docker.sock"
+{{- end -}}
+
+{{- define "docker_volume_mounts" -}}
+- name: cgroup
+ mountPath: "/sys/fs/cgroup"
+- name: runc
+ mountPath: "/run/runc"
+- name: dockersock
+ mountPath: "/var/run/docker.sock"
+- name: dockerrootdir
+ mountPath: "/containers"
+{{- end -}}
+
+{{- define "docker_pull_runtimes" -}}
+- name: docker-pull-runtimes
+ imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
+ image: {{ .Values.invoker.pullRuntimesImage | quote }}
+ volumeMounts:
+ - name: dockersock
+ mountPath: "/var/run/docker.sock"
+ env:
+ # action runtimes
+ - name: "RUNTIMES_MANIFEST"
+ value: {{ template "runtimes_manifest" . }}
+{{- end -}}
+
diff --git a/helm/templates/invoker.yaml b/helm/templates/invoker.yaml
index d5ac64e..d1c897f 100644
--- a/helm/templates/invoker.yaml
+++ b/helm/templates/invoker.yaml
@@ -1,53 +1,52 @@
# Licensed to the Apache Software Foundation (ASF) under one or more contributor
# license agreements; and to You under the Apache License, Version 2.0.
+---
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
apiVersion: extensions/v1beta1
kind: DaemonSet
+{{- else if eq .Values.invoker.containerFactory.impl "kubernetes" -}}
+apiVersion: apps/v1beta1
+kind: StatefulSet
+{{- end }}
metadata:
name: {{ .Values.invoker.name | quote }}
namespace: {{ .Release.Namespace | quote }}
labels:
name: {{ .Values.invoker.name | quote }}
spec:
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+ replicas: {{ .Values.invoker.containerFactory.kubernetes.replicaCount }}
+{{- end }}
template:
metadata:
labels:
name: {{ .Values.invoker.name | quote }}
spec:
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+ serviceAccountName: {{ .Values.invoker.name | quote }}
+{{- end }}
restartPolicy: {{ .Values.invoker.restartPolicy }}
affinity:
{{ include "affinity.invoker" . | indent 8 }}
+{{- if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+{{ include "affinity.selfAntiAffinity" ( .Values.invoker.name | quote ) | indent 8 }}
+{{- end }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
volumes:
- - name: cgroup
- hostPath:
- path: "/sys/fs/cgroup"
- - name: runc
- hostPath:
- path: "/run/runc"
- - name: dockerrootdir
- hostPath:
- path: "/var/lib/docker/containers"
- - name: dockersock
- hostPath:
- path: "/var/run/docker.sock"
+{{ include "docker_volumes" . | indent 6 }}
+{{- end }}
initContainers:
# Wait for a controller to be up (which implies kafka, zookeeper, couchdb are all up as well).
{{ include "readiness.waitForController" . | indent 6 }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
# Pull images for all default runtimes before starting invoker
- - name: docker-pull-runtimes
- imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
- image: openwhisk/kube-docker-pull
- volumeMounts:
- - name: dockersock
- mountPath: "/var/run/docker.sock"
- env:
- # action runtimes
- - name: "RUNTIMES_MANIFEST"
- value: {{ template "runtimes_manifest" . }}
+{{ include "docker_pull_runtimes" . | indent 6 }}
+{{- end }}
containers:
- name: {{ .Values.invoker.name | quote }}
@@ -56,7 +55,7 @@ spec:
command: [ "/bin/bash", "-c", "COMPONENT_NAME=$(hostname | cut -d'-' -f2) /init.sh"]
env:
- name: "PORT"
- value: "8080"
+ value: {{ .Values.invoker.port | quote }}
- name: "WHISK_API_HOST_NAME"
valueFrom:
configMapKeyRef:
@@ -75,19 +74,26 @@ spec:
- name: "DOCKER_REGISTRY"
value: ""
- # Invoker name is name of the Kube node when using DaemonSet
+ # Invoker name is the name of the node (DaemonSet) or pod (StatefulSet)
- name: "INVOKER_NAME"
valueFrom:
fieldRef:
- fieldPath: spec.nodeName
+ fieldPath: {{ if eq .Values.invoker.containerFactory.impl "docker" }} spec.nodeName {{ else }} metadata.name {{ end }}
# Java options
- name: "JAVA_OPTS"
- value: "-Xmx2g"
+ value: {{ .Values.invoker.jvmOptions | quote }}
# Invoker options
- name: "INVOKER_OPTS"
- value: {{ .Values.invoker.options | quote }}
+ value: "{{ .Values.invoker.options }} {{ if eq .Values.invoker.containerFactory.impl "docker" }} -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.docker.DockerContainerFactoryProvider {{ else }} -Dkubernetes.master=https://$KUBERNETES_SERVICE_HOST -Dwhisk.spi.ContainerFactoryProvider=whisk.core.containerpool.kubernetes.KubernetesContainerFactoryProvider {{ end }}"
+
+{{ if .Values.invoker.containerFactory.kubernetes.agent.enabled }}
+ - name: "CONFIG_whisk_kubernetes_invokerAgent_enabled"
+ value: "TRUE"
+ - name: "CONFIG_whisk_kubernetes_invokerAgent_port"
+ value: {{ .Values.invoker.containerFactory.kubernetes.agent.port | quote }}
+{{ end }}
# action runtimes
- name: "RUNTIMES_MANIFEST"
@@ -114,13 +120,56 @@ spec:
ports:
- name: invoker
- containerPort: 8080
+ containerPort: {{ .Values.invoker.port }}
+{{- if eq .Values.invoker.containerFactory.impl "docker" }}
+ volumeMounts:
+{{ include "docker_volume_mounts" . | indent 8 }}
+{{- end }}
+
+{{ if .Values.invoker.containerFactory.kubernetes.agent.enabled }}
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+ name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+ namespace: {{ .Release.Namespace | quote }}
+ labels:
+ name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+spec:
+ template:
+ metadata:
+ labels:
+ name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+ spec:
+ restartPolicy: Always
+ hostNetwork: true
+
+ # TODO: disabled affinity until user-action pods are
+ # created by KubernetesContainerFacotry with the
+ # same affinity rules.
+ # Requires extension to upstream kube java client
+ # run only on nodes labeled with openwhisk-role=invoker
+
+ volumes:
+{{ include "docker_volumes" . | indent 6 }}
+ - name: userlogs
+ emptyDir: {}
+
+ initContainers:
+ # Pull images for all default runtimes before starting invoker
+{{ include "docker_pull_runtimes" . | indent 6 }}
+
+ containers:
+ - name: {{ .Values.invoker.containerFactory.kubernetes.agent.name | quote }}
+ imagePullPolicy: {{ .Values.invoker.imagePullPolicy | quote }}
+ image: {{ .Values.invoker.containerFactory.kubernetes.agent.image }}
+ securityContext:
+ privileged: true
+ ports:
+ - name: agent
+ containerPort: {{ .Values.invoker.containerFactory.kubernetes.agent.port }}
volumeMounts:
- - name: cgroup
- mountPath: "/sys/fs/cgroup"
- - name: runc
- mountPath: "/run/runc"
- - name: dockersock
- mountPath: "/var/run/docker.sock"
- - name: dockerrootdir
- mountPath: "/containers"
+{{ include "docker_volume_mounts" . | indent 8 }}
+ - name: userlogs
+ mountPath: "/action-logs"
+{{- end }}
diff --git a/helm/templates/rolebindings.yaml b/helm/templates/rolebindings.yaml
index 298bc5b..b8b4bca 100644
--- a/helm/templates/rolebindings.yaml
+++ b/helm/templates/rolebindings.yaml
@@ -22,3 +22,48 @@ subjects:
- kind: ServiceAccount
name: ow-core
namespace: {{ .Release.Namespace | quote }}
+
+
+{{ if eq .Values.invoker.containerFactory.impl "kubernetes" }}
+# When using KubernetesContainerFactory, invoker pods need extensive
+# permissions to manage pods and deployments. The ability to create
+# pods can enable privilege escalation attacks, so restrict it to a
+# ServiceAccount that is only used for the invokers and only defined
+# when using KubernetesContainerFactory.
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+rules:
+- apiGroups: ["extensions"]
+ resources: ["deployments"]
+ verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+ resources: ["pods"]
+ verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: [""]
+ resources: ["pods/log"]
+ verbs: ["get", "list"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1beta1
+metadata:
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+subjects:
+- kind: ServiceAccount
+ namespace: {{ .Release.Namespace | quote }}
+ name: {{ .Values.invoker.name | quote }}
+roleRef:
+ kind: Role
+ name: {{ .Values.invoker.name | quote }}
+ apiGroup: rbac.authorization.k8s.io
+{{- end }}
diff --git a/helm/values.yaml b/helm/values.yaml
index d74d452..5944d73 100644
--- a/helm/values.yaml
+++ b/helm/values.yaml
@@ -94,7 +94,19 @@ invoker:
image: "openwhisk/invoker"
imagePullPolicy: "IfNotPresent"
restartPolicy: "Always"
+ port: 8080
+ pullRuntimesImage: "openwhisk/kube-docker-pull"
options: ""
+ jvmOptions: "-Xmx512M"
+ containerFactory:
+ impl: "docker"
+ kubernetes:
+ replicaCount: 1
+ agent:
+ name: "invoker-agent"
+ image: "openwhisk/kube-invoker-agent"
+ enabled: false
+ port: 3233
# API Gateway configurations
apigw:
diff --git a/tools/travis/build-helm.sh b/tools/travis/build-helm.sh
index bb3ce6b..cdb39a5 100755
--- a/tools/travis/build-helm.sh
+++ b/tools/travis/build-helm.sh
@@ -154,6 +154,13 @@ whisk:
api_host: $WSK_HOST:$WSK_PORT
runtimes: "runtimes-minimal-travis.json"
+invoker:
+ containerFactory:
+ impl: $OW_CONTAINER_FACTORY
+ kubernetes:
+ agent:
+ enabled: true
+
nginx:
httpsNodePort: $WSK_PORT
EOF
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index 626b1d2..48f1348 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -234,7 +234,7 @@ pushd kubernetes/invoker
echo "Deploying invoker using DockerContainerFactory"
kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-dcf.env
kubectl apply -f invoker-dcf.yml
- elif [ "$OW_CONTAINER_FACTORY" = "kube" ]; then
+ elif [ "$OW_CONTAINER_FACTORY" = "kubernetes" ]; then
echo "Deploying invoker using KubernetesContainerFactory"
kubectl -n openwhisk create cm invoker.config --from-env-file=invoker-k8scf.env
kubectl apply -f invoker-agent.yml
diff --git a/tools/travis/setup.sh b/tools/travis/setup.sh
index 251fde8..7ecc8a6 100755
--- a/tools/travis/setup.sh
+++ b/tools/travis/setup.sh
@@ -59,7 +59,7 @@ mkdir $HOME/.kube || true
touch $HOME/.kube/config
export KUBECONFIG=$HOME/.kube/config
-sudo -E /usr/local/bin/minikube start --vm-driver=none --kubernetes-version=$TRAVIS_KUBE_VERSION
+sudo -E /usr/local/bin/minikube start --vm-driver=none --extra-config=apiserver.Authorization.Mode=RBAC --kubernetes-version=$TRAVIS_KUBE_VERSION
# Wait until we have a ready node in minikube
TIMEOUT=0
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services