You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ra...@apache.org on 2018/01/16 19:01:15 UTC
[incubator-openwhisk-deploy-kube] branch master updated: Specify
pod/node scheduling affinities (#151)
This is an automated email from the ASF dual-hosted git repository.
rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git
The following commit(s) were added to refs/heads/master by this push:
new 9ab697a Specify pod/node scheduling affinities (#151)
9ab697a is described below
commit 9ab697a34be6deeb802825533179a2ef8f6f282d
Author: David Grove <dg...@users.noreply.github.com>
AuthorDate: Tue Jan 16 14:01:12 2018 -0500
Specify pod/node scheduling affinities (#151)
Specify pod and node affinities for Kubernetes scheduler
to implement two basic policies:
1. Do not schedule anything except the invoker pod
on nodes labeled openwhisk-role=invoker.
2. Do not allow multiple instances of the same pod
to be scheduled on the same node (HA -- replicated
pods should be run on distinct worker nodes).
Also weakly prefer to co-locate kafka and zookeeper pods
on the same node to optimize their communication.
Fixes #99.
---
kubernetes/apigateway/apigateway.yml | 22 ++++++++++++++++++++++
kubernetes/controller/controller.yml | 22 ++++++++++++++++++++++
kubernetes/couchdb/couchdb.yml | 23 +++++++++++++++++++++++
kubernetes/invoker/README.md | 6 +++---
kubernetes/invoker/invoker.yml | 13 +++++++++++--
kubernetes/kafka/kafka.yml | 34 ++++++++++++++++++++++++++++++++++
kubernetes/nginx/nginx.yml | 24 ++++++++++++++++++++++++
kubernetes/zookeeper/zookeeper.yml | 22 ++++++++++++++++++++++
tools/travis/build.sh | 2 +-
9 files changed, 162 insertions(+), 6 deletions(-)
diff --git a/kubernetes/apigateway/apigateway.yml b/kubernetes/apigateway/apigateway.yml
index 98d8d3b..119bfbb 100644
--- a/kubernetes/apigateway/apigateway.yml
+++ b/kubernetes/apigateway/apigateway.yml
@@ -15,6 +15,28 @@ spec:
spec:
restartPolicy: Always
+ affinity:
+ # prefer to not run on an invoker node (only prefer because of single node clusters)
+ nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - invoker
+ # do not allow more than 1 apigateway instance to run on a node
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - apigateway
+ topologyKey: "kubernetes.io/hostname"
+
containers:
- name: redis
imagePullPolicy: IfNotPresent
diff --git a/kubernetes/controller/controller.yml b/kubernetes/controller/controller.yml
index 801379d..a7a19ce 100644
--- a/kubernetes/controller/controller.yml
+++ b/kubernetes/controller/controller.yml
@@ -16,6 +16,28 @@ spec:
spec:
restartPolicy: Always
+ affinity:
+ # prefer to not run on an invoker node (only prefer because of single node clusters)
+ nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - invoker
+ # do not allow more than 1 controller instance to run on a node
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - controller
+ topologyKey: "kubernetes.io/hostname"
+
containers:
- name: controller
imagePullPolicy: Always
diff --git a/kubernetes/couchdb/couchdb.yml b/kubernetes/couchdb/couchdb.yml
index 76b349e..a06aea7 100644
--- a/kubernetes/couchdb/couchdb.yml
+++ b/kubernetes/couchdb/couchdb.yml
@@ -32,6 +32,29 @@ spec:
name: couchdb
spec:
restartPolicy: Always
+
+ affinity:
+ # prefer to not run on an invoker node (only prefer because of single node clusters)
+ nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - invoker
+ # do not allow more than 1 couchdb instance to run on a given node
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - couchdb
+ topologyKey: "kubernetes.io/hostname"
+
containers:
- name: couchdb
imagePullPolicy: Always
diff --git a/kubernetes/invoker/README.md b/kubernetes/invoker/README.md
index 3632b84..9d420cf 100644
--- a/kubernetes/invoker/README.md
+++ b/kubernetes/invoker/README.md
@@ -25,8 +25,8 @@ With the defaults in the current `invoker.yml`, you can setup a
node to run only Invoker pods with:
```
-kubectl label nodes [node name] openwhisk=invoker
-$ kubectl label nodes 127.0.0.1 openwhisk=invoker
+kubectl label nodes [node name] openwhisk-role=invoker
+$ kubectl label nodes 127.0.0.1 openwhisk-role=invoker
```
Once the invoker label is applied, you can create the invokers with:
@@ -49,7 +49,7 @@ section below.
# Troubleshooting
## No invokers are deployed
-Verify that you actually have nodes with the label openwhisk=invoker.
+Verify that you actually have at least one node with the label openwhisk-role=invoker.
## Kubernetes Host Linux Versions
diff --git a/kubernetes/invoker/invoker.yml b/kubernetes/invoker/invoker.yml
index c9739c9..6e7031f 100644
--- a/kubernetes/invoker/invoker.yml
+++ b/kubernetes/invoker/invoker.yml
@@ -13,8 +13,17 @@ spec:
name: invoker
spec:
restartPolicy: Always
- nodeSelector:
- openwhisk: "invoker"
+
+ # run only on nodes labeled with openwhisk-role=invoker
+ affinity:
+ nodeAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ nodeSelectorTerms:
+ - matchExpressions:
+ - key: openwhisk-role
+ operator: In
+ values:
+ - invoker
volumes:
- name: cgroup
diff --git a/kubernetes/kafka/kafka.yml b/kubernetes/kafka/kafka.yml
index 9d6b1d4..bc4ca09 100644
--- a/kubernetes/kafka/kafka.yml
+++ b/kubernetes/kafka/kafka.yml
@@ -15,6 +15,40 @@ spec:
spec:
restartPolicy: Always
+ affinity:
+ # prefer to not run on an invoker node (only prefer because of single node clusters)
+ nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - invoker
+ # do not allow more than 1 kafka instance to run on a given node
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - kafka
+ topologyKey: "kubernetes.io/hostname"
+ # prefer to co-locate with a zookeeper pod since we communicate frequently
+ podAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 10
+ podAffinityTerm:
+ labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - zookeeper
+ topologyKey: "kubernetes.io/hostname"
+
containers:
- name: kafka
imagePullPolicy: IfNotPresent
diff --git a/kubernetes/nginx/nginx.yml b/kubernetes/nginx/nginx.yml
index 486a66d..e69c77f 100644
--- a/kubernetes/nginx/nginx.yml
+++ b/kubernetes/nginx/nginx.yml
@@ -14,6 +14,29 @@ spec:
name: nginx
spec:
restartPolicy: Always
+
+ affinity:
+ # prefer to not run on an invoker node (only prefer because of single node clusters)
+ nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - invoker
+ # do not allow more than 1 nginx instance to run on a given node
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - nginx
+ topologyKey: "kubernetes.io/hostname"
+
volumes:
- name: nginx-certs
secret:
@@ -23,6 +46,7 @@ spec:
name: nginx
- name: logs
emptyDir: {}
+
containers:
- name: nginx
imagePullPolicy: IfNotPresent
diff --git a/kubernetes/zookeeper/zookeeper.yml b/kubernetes/zookeeper/zookeeper.yml
index 26422a5..15f1b74 100644
--- a/kubernetes/zookeeper/zookeeper.yml
+++ b/kubernetes/zookeeper/zookeeper.yml
@@ -14,6 +14,28 @@ spec:
spec:
restartPolicy: Always
+ affinity:
+ # prefer to not run on an invoker node (only prefer because of single node clusters)
+ nodeAffinity:
+ preferredDuringSchedulingIgnoredDuringExecution:
+ - weight: 100
+ preference:
+ matchExpressions:
+ - key: openwhisk-role
+ operator: NotIn
+ values:
+ - invoker
+ # do not allow more than 1 zookeeper instance to run on a given node
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: name
+ operator: In
+ values:
+ - zookeeper
+ topologyKey: "kubernetes.io/hostname"
+
containers:
- name: zookeeper
image: zookeeper:3.4
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index 35cb378..a6de7b5 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -139,7 +139,7 @@ cd $ROOTDIR
# Label invoker nodes (needed for daemonset-based invoker deployment)
echo "Labeling invoker node"
-kubectl label nodes --all openwhisk=invoker
+kubectl label nodes --all openwhisk-role=invoker
kubectl describe nodes
# Initial cluster setup
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].