[incubator-openwhisk] branch master updated: Add tests to check high-availability of the controller. (#2661)

[ma...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git


The following commit(s) were added to refs/heads/master by this push:
     new 66c9dbc  Add tests to check high-availability of the controller. (#2661)
66c9dbc is described below

commit 66c9dbc094234c6851731c2f858ea9e5977b5409
Author: Christian Bickel <gi...@cbickel.de>
AuthorDate: Mon Sep 11 10:14:21 2017 +0200

    Add tests to check high-availability of the controller. (#2661)
---
 tests/src/test/scala/ha/ShootComponentsTests.scala | 147 +++++++++++++++++++++
 1 file changed, 147 insertions(+)

diff --git a/tests/src/test/scala/ha/ShootComponentsTests.scala b/tests/src/test/scala/ha/ShootComponentsTests.scala
new file mode 100644
index 0000000..4b28ded
--- /dev/null
+++ b/tests/src/test/scala/ha/ShootComponentsTests.scala
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ha
+
+import java.io.File
+import java.time.Instant
+
+import scala.concurrent.Future
+import scala.concurrent.duration.DurationInt
+import scala.util.Try
+
+import org.junit.runner.RunWith
+import org.scalatest.FlatSpec
+import org.scalatest.Matchers
+import org.scalatest.concurrent.ScalaFutures
+import org.scalatest.junit.JUnitRunner
+
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.HttpRequest
+import akka.http.scaladsl.model.StatusCodes
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
+import common.TestUtils
+import common.WhiskProperties
+import common.Wsk
+import common.WskActorSystem
+import common.WskProps
+import common.WskTestHelpers
+import whisk.core.WhiskConfig
+import whisk.utils.retry
+
+@RunWith(classOf[JUnitRunner])
+class ShootComponentsTests extends FlatSpec with Matchers with WskTestHelpers with ScalaFutures with WskActorSystem {
+
+  implicit val wskprops = WskProps()
+  val wsk = new Wsk
+  val defaultAction = Some(TestUtils.getTestActionFilename("hello.js"))
+
+  implicit val materializer = ActorMaterializer()
+  implicit val testConfig = PatienceConfig(1.minute)
+
+  val controller0DockerHost = WhiskProperties.getBaseControllerHost() + ":" + WhiskProperties.getProperty(
+    WhiskConfig.dockerPort)
+
+  def restartComponent(host: String, component: String) = {
+    def file(path: String) = Try(new File(path)).filter(_.exists).map(_.getAbsolutePath).toOption
+    val docker = (file("/usr/bin/docker") orElse file("/usr/local/bin/docker")).getOrElse("docker")
+
+    val cmd = Seq(docker, "--host", host, "restart", component)
+    println(s"Running command: ${cmd.mkString(" ")}")
+
+    TestUtils.runCmd(0, new File("."), cmd: _*)
+  }
+
+  def ping(host: String, port: Int) = {
+    val response = Try { Http().singleRequest(HttpRequest(uri = s"http://$host:$port/ping")).futureValue }.toOption
+
+    response.map { res =>
+      (res.status, Unmarshal(res).to[String].futureValue)
+    }
+  }
+
+  def isControllerAlive(instance: Int): Boolean = {
+    require(instance >= 0 && instance < 2, "Controller instance not known.")
+
+    val host = WhiskProperties.getProperty("controller.hosts").split(",")(instance)
+    val port = WhiskProperties.getControllerBasePort + instance
+
+    val res = ping(host, port)
+    res == Some((StatusCodes.OK, "pong"))
+  }
+
+  def doRequests(amount: Int, actionName: String): Seq[(Int, Int)] = {
+    (0 until amount).map { i =>
+      val start = Instant.now
+
+      // Do POSTs and GETs
+      val invokeExit = Future { wsk.action.invoke(actionName, expectedExitCode = TestUtils.DONTCARE_EXIT).exitCode }
+      val getExit = Future { wsk.action.get(actionName, expectedExitCode = TestUtils.DONTCARE_EXIT).exitCode }
+
+      println(s"Done rerquests with responses: invoke: ${invokeExit.futureValue} and get: ${getExit.futureValue}")
+
+      // Do at most one action invocation per second to avoid getting 429s. (60 req/min - limit)
+      val wait = 1000 - (Instant.now.toEpochMilli - start.toEpochMilli)
+      Thread.sleep(if (wait < 0) 0L else if (wait > 1000) 1000L else wait)
+      (invokeExit.futureValue, getExit.futureValue)
+    }
+  }
+
+  behavior of "Controllers hot standby"
+
+  it should "use controller1 if controller0 goes down" in withAssetCleaner(wskprops) { (wp, assetHelper) =>
+    if (WhiskProperties.getProperty(WhiskConfig.controllerInstances).toInt >= 2) {
+      val actionName = "shootcontroller"
+
+      assetHelper.withCleaner(wsk.action, actionName) { (action, _) =>
+        action.create(actionName, defaultAction)
+      }
+
+      // Produce some load on the system for 100 seconds (each second one request). Kill the controller after 4 requests
+      val totalRequests = 100
+
+      val requestsBeforeRestart = doRequests(4, actionName)
+
+      // Kill the controller
+      restartComponent(controller0DockerHost, "controller0")
+      // Wait until down
+      retry({
+        isControllerAlive(0) shouldBe false
+      }, 100, Some(100.milliseconds))
+      // Check that second controller is still up
+      isControllerAlive(1) shouldBe true
+
+      val requestsAfterRestart = doRequests(totalRequests - 4, actionName)
+
+      val requests = requestsBeforeRestart ++ requestsAfterRestart
+
+      val unsuccessfulInvokes = requests.map(_._1).count(_ != TestUtils.SUCCESS_EXIT)
+      // Allow 3 failures for the 90 seconds
+      unsuccessfulInvokes should be <= 3
+
+      val unsuccessfulGets = requests.map(_._2).count(_ != TestUtils.SUCCESS_EXIT)
+      // Only allow 1 failure in GET requests, because they are idempotent and they should be passed to the next controller if one crashes
+      unsuccessfulGets shouldBe 0
+
+      // Check that both controllers are up
+      // controller0
+      isControllerAlive(0) shouldBe true
+      //controller1
+      isControllerAlive(1) shouldBe true
+    }
+  }
+}

-- 
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].