You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by cb...@apache.org on 2017/06/26 15:33:12 UTC
[incubator-openwhisk] branch master updated: Back-off threshold and
choose home invoker as default. (#2417)
This is an automated email from the ASF dual-hosted git repository.
cbickel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 9bb2617 Back-off threshold and choose home invoker as default. (#2417)
9bb2617 is described below
commit 9bb26173e71e2c71667810970e900c291d5a8ac3
Author: Markus Thömmes <ma...@me.com>
AuthorDate: Mon Jun 26 17:33:09 2017 +0200
Back-off threshold and choose home invoker as default. (#2417)
* Back-off threshold and choose home invoker as default.
The loadbalancer falls back to picking the home invoker for the action iff all invokers are loaded above a defined threshold. That threshold is backed off 3 times to prevent a "all hell breaks loose" scenario in a sustained high load.
---
.../core/loadBalancer/LoadBalancerService.scala | 25 ++++++++++++----------
.../test/LoadBalancerServiceObjectTests.scala | 25 +++++++++++++++++-----
2 files changed, 34 insertions(+), 16 deletions(-)
diff --git a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
index 5390a3a..d2e8b64 100644
--- a/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
+++ b/core/controller/src/main/scala/whisk/core/loadBalancer/LoadBalancerService.scala
@@ -336,8 +336,8 @@ object LoadBalancerService {
/**
* Scans through all invokers and searches for an invoker, that has a queue length
- * below the defined threshold. Iff no "underloaded" invoker was found it will
- * default to the least loaded invoker in the list.
+ * below the defined threshold. The threshold is subject to a 3 times back off. Iff
+ * no "underloaded" invoker was found it will default to the home invoker.
*
* @param availableInvokers a list of available (healthy) invokers to search in
* @param activationsPerInvoker a map of the number of outstanding activations per invoker
@@ -359,30 +359,33 @@ object LoadBalancerService {
val step = stepSizes(hash % stepSizes.size)
@tailrec
- def search(targetInvoker: Int, seenInvokers: Int): A = {
+ def search(targetInvoker: Int, iteration: Int = 1): A = {
// map the computed index to the actual invoker index
val invokerName = availableInvokers(targetInvoker)
// send the request to the target invoker if it has capacity...
- if (activationsPerInvoker.get(invokerName).getOrElse(0) < invokerBusyThreshold) {
+ if (activationsPerInvoker.get(invokerName).getOrElse(0) < invokerBusyThreshold * iteration) {
invokerName
} else {
- // ... otherwise look for a less loaded invoker by stepping through a pre computed
+ // ... otherwise look for a less loaded invoker by stepping through a pre-computed
// list of invokers; there are two possible outcomes:
// 1. the search lands on a new invoker that has capacity, choose it
// 2. walked through the entire list and found no better invoker than the
- // "home invoker", choose the least loaded invoker
+ // "home invoker", force the home invoker
val newTarget = (targetInvoker + step) % numInvokers
- if (newTarget == homeInvoker || seenInvokers > numInvokers) {
- // fall back to the invoker with the least load.
- activationsPerInvoker.minBy(_._2)._1
+ if (newTarget == homeInvoker) {
+ if (iteration < 3) {
+ search(newTarget, iteration + 1)
+ } else {
+ availableInvokers(homeInvoker)
+ }
} else {
- search(newTarget, seenInvokers + 1)
+ search(newTarget, iteration)
}
}
}
- Some(search(homeInvoker, 0))
+ Some(search(homeInvoker))
} else {
None
}
diff --git a/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala b/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala
index 4f5433c..9b8bafc 100644
--- a/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala
+++ b/tests/src/test/scala/whisk/core/loadBalancer/test/LoadBalancerServiceObjectTests.scala
@@ -112,15 +112,30 @@ class LoadBalancerServiceObjectTests extends FlatSpec with Matchers {
1, hash) shouldBe Some(hashInto(invs, hash + step + step))
}
- it should "choose the least loaded invoker if all invokers are overloaded to begin with" in {
+ it should "multiply its threshold in 3 iterations to find an invoker with a good warm-chance" in {
val invokerCount = 3
val invs = invokers(invokerCount)
- val hash = 1
+ val hash = 0 // home is 0, stepsize is 1
+
+ // even though invoker1 is not the home invoker in this case, it gets chosen over
+ // the others because it's the first one encountered by the iteration mechanism to be below
+ // the threshold of 3 * 16 invocations
+ LoadBalancerService.schedule(
+ invs,
+ Map("invoker0" -> 33, "invoker1" -> 36, "invoker2" -> 33),
+ 16,
+ hash) shouldBe Some("invoker0")
+ }
+
+ it should "choose the home invoker if all invokers are overloaded even above the muliplied threshold" in {
+ val invokerCount = 3
+ val invs = invokers(invokerCount)
+ val hash = 0 // home is 0, stepsize is 1
LoadBalancerService.schedule(
invs,
- Map("invoker0" -> 3, "invoker1" -> 3, "invoker2" -> 2),
- 1,
- hash) shouldBe Some("invoker2")
+ Map("invoker0" -> 51, "invoker1" -> 50, "invoker2" -> 49),
+ 16,
+ hash) shouldBe Some("invoker0")
}
}
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].