[GitHub] markusthoemmes closed pull request #3556: Add latency test with gatling.

[GitBox <gi...@apache.org>]

markusthoemmes closed pull request #3556: Add latency test with gatling.
URL: https://github.com/apache/incubator-openwhisk/pull/3556
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/.travis.yml b/.travis.yml
index 7fc1587385..f5b0fd8e38 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -65,5 +65,6 @@ jobs:
         - TERM=dumb ./performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" 2m
         - TERM=dumb ./performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" 4 2 2m
         - OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-ApiV1Simulation
+        - OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" ./gradlew gatlingRun-LatencySimulation
       env:
         - DESCRIPTION="Execute wrk-performance test suite."
diff --git a/performance/README.md b/performance/README.md
index f089021e13..e8d21646dd 100644
--- a/performance/README.md
+++ b/performance/README.md
@@ -61,3 +61,33 @@ You can run the simulation with (in OPENWHISK_HOME)
 ```
 OPENWHISK_HOST="openwhisk.mydomain.com" CONNECTIONS="10" REQUESTS_PER_SEC="50" ./gradlew gatlingRun-ApiV1Simulation
 ```
+
+##### Latency Simulation
+
+This simulation creates actions of the following four kinds: `nodejs:default`, `swift:default`, `java:default` and
+`python:default`.
+Afterwards the action is invoked once. This is the cold-start and will not be part of the thresholds.
+Next, the action will be invoked 100 times blocking and one after each other. The last step is, that the action will be deleted.
+
+Once one language is finished, the next kind will be taken. They are not running in parallel. There are never more than
+1 activations in the system, as we only want to meassure latency of warm activations.
+As all actions are invoked blocking and only one action is in the system, it doesn't matter how many controllers
+and invokers are deployed. If several controllers or invokers are deployed, all controllers send the activation
+always to the same invoker.
+
+The comparison of the thresholds is against the mean response times of the warm activations.
+
+Available environment variables:
+
+```
+OPENWHISK_HOST          (required)
+API_KEY                 (required, format: UUID:KEY)
+MEAN_RESPONSE_TIME      (required)
+MAX_MEAN_RESPONSE_TIME  (default: MEAN_RESPONSE_TIME)
+EXCLUDED_KINDS          (default: "", format: "python:default,java:default,swift:default")
+```
+
+You can run the simulation with (in OPENWHISK_HOME)
+```
+OPENWHISK_HOST="openwhisk.mydomain.com" MEAN_RESPONSE_TIME="20" API_KEY="UUID:KEY" ./gradlew gatlingRun-LatencySimulation
+```
diff --git a/performance/gatling_tests/build.gradle b/performance/gatling_tests/build.gradle
index 7e8f1ad40d..9242fb45b7 100644
--- a/performance/gatling_tests/build.gradle
+++ b/performance/gatling_tests/build.gradle
@@ -1,5 +1,5 @@
 plugins {
-    id "com.github.lkishalmi.gatling" version "0.7.1"
+    id "com.github.lkishalmi.gatling" version "0.7.2"
 }
 
 apply plugin: 'eclipse'
@@ -10,8 +10,8 @@ repositories {
 }
 
 dependencies {
-    compile "org.scala-lang:scala-library:${gradle.scala.version}"
-    compile "io.gatling.highcharts:gatling-charts-highcharts:2.2.5"
+    gatling "io.spray:spray-json_2.12:1.3.4"
+    gatling "commons-io:commons-io:2.6"
 }
 
 tasks.withType(ScalaCompile) {
diff --git a/performance/gatling_tests/src/gatling/resources/data/javaAction.jar b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar
new file mode 100644
index 0000000000..17229eb950
Binary files /dev/null and b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar differ
diff --git a/performance/gatling_tests/src/gatling/resources/data/javaAction.java b/performance/gatling_tests/src/gatling/resources/data/javaAction.java
new file mode 100644
index 0000000000..56da8db746
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/javaAction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Build the jar with the following commands:
+//
+// javac -cp gson-2.8.2.jar JavaAction.java
+// jar cvf javaAction.jar JavaAction.class
+
+import com.google.gson.JsonObject;
+
+public class JavaAction {
+    public static JsonObject main(JsonObject args) {
+        String text;
+
+        try {
+            text = args.getAsJsonPrimitive("text").getAsString();
+        } catch(Exception e) {
+            text = "stranger";
+        }
+
+        JsonObject response = new JsonObject();
+        System.out.println("Hello " + text + "!");
+        response.addProperty("payload", "Hello " + text + "!");
+        return response;
+    }
+}
diff --git a/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js
new file mode 100644
index 0000000000..8c71017324
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js
@@ -0,0 +1,5 @@
+function main(params) {
+    var greeting = "Hello" + (params.text || "stranger") + "!";
+    console.log(greeting);
+    return { payload: greeting };
+}
diff --git a/performance/gatling_tests/src/gatling/resources/data/pythonAction.py b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py
new file mode 100644
index 0000000000..c26b0605ef
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py
@@ -0,0 +1,9 @@
+import sys
+def main(dict):
+    if 'text' in dict:
+        text = dict['text']
+    else:
+        text = "stranger"
+    greeting = "Hello " + text + "!"
+    print(greeting)
+    return {"payload": greeting}
diff --git a/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift
new file mode 100644
index 0000000000..3f52f66a01
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift
@@ -0,0 +1,9 @@
+func main(args: [String:Any]) -> [String:Any] {
+    if let text = args["text"] as? String {
+        print("Hello " + text + "!")
+        return [ "payload" : "Hello " + text + "!" ]
+    } else {
+        print("Hello stranger!")
+        return [ "payload" : "Hello stranger!" ]
+    }
+}
diff --git a/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
new file mode 100644
index 0000000000..9ec41f074d
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
+import extension.whisk.Predef._
+import io.gatling.core.Predef._
+import io.gatling.core.session.Expression
+import io.gatling.core.util.Resource
+import org.apache.commons.io.FileUtils
+
+class LatencySimulation extends Simulation {
+  // Specify parameters for the run
+  val host = sys.env("OPENWHISK_HOST")
+
+  // Specify authentication
+  val Array(uuid, key) = sys.env("API_KEY").split(":")
+
+  // Specify thresholds
+  val meanResponseTime: Int = sys.env("MEAN_RESPONSE_TIME").toInt
+  val maximalMeanResponseTime: Int = sys.env.getOrElse("MAX_MEAN_RESPONSE_TIME", meanResponseTime.toString).toInt
+
+  // Exclude runtimes
+  val excludedKinds: Seq[String] = sys.env.getOrElse("EXCLUDED_KINDS", "").split(",")
+
+  // Generate the OpenWhiskProtocol
+  val openWhiskProtocol = openWhisk.apiHost(host)
+
+  /**
+   * Generate a list of actions to execute. The list is a tuple of (kind, code, actionName, main)
+   * `kind` is needed to create the action
+   * `code` is loaded form the files located in `resources/data`
+   * `actionName` is the name of the action in OpenWhisk
+   * `main` is only needed for java. This is the name of the class where the main method is located.
+   */
+  val actions: Seq[(String, String, String, String)] = Map(
+    "nodejs:default" -> (FileUtils
+      .readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8), "latencyTest_node", ""),
+    "python:default" -> (FileUtils
+      .readFileToString(Resource.body("pythonAction.py").get.file, StandardCharsets.UTF_8), "latencyTest_python", ""),
+    "swift:default" -> (FileUtils
+      .readFileToString(Resource.body("swiftAction.swift").get.file, StandardCharsets.UTF_8), "latencyTest_swift", ""),
+    "java:default" -> (Base64.getEncoder.encodeToString(
+      FileUtils.readFileToByteArray(Resource.body("javaAction.jar").get.file)), "latencyTest_java", "JavaAction"))
+    .filterNot(e => excludedKinds.contains(e._1))
+    .map {
+      case (kind, (code, name, main)) =>
+        (kind, code, name, main)
+    }
+    .toSeq
+
+  // Define scenario
+  val test = scenario("Invoke one action after each other to test latency")
+    .foreach(actions, "action") {
+      val code: Expression[String] = "${action._2}"
+      exec(
+        openWhisk("Create ${action._1} action")
+          .authenticate(uuid, key)
+          .action("${action._3}")
+          .create(code, "${action._1}", "${action._4}"))
+        .exec(openWhisk("Cold ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke())
+        .repeat(100) {
+          exec(openWhisk("Warm ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke())
+        }
+        .exec(openWhisk("Delete ${action._1} action").authenticate(uuid, key).action("${action._3}").delete())
+    }
+
+  val testSetup = setUp(test.inject(atOnceUsers(1)))
+    .protocols(openWhiskProtocol)
+
+  actions
+    .map { case (kind, _, _, _) => s"Warm $kind invocation" }
+    .foldLeft(testSetup) { (agg, cur) =>
+      // One failure will make the build yellow
+      agg
+        .assertions(details(cur).responseTime.mean.lte(meanResponseTime))
+        .assertions(details(cur).responseTime.mean.lt(maximalMeanResponseTime))
+        // Mark the build yellow, if there are failed requests. And red if both conditions fail.
+        .assertions(details(cur).failedRequests.count.is(0))
+        .assertions(details(cur).failedRequests.percent.lte(0.1))
+    }
+}
diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
index ef6d2efde9..56dc6ceafb 100644
--- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
+++ b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
@@ -23,12 +23,100 @@ import io.gatling.core.action.builder.ActionBuilder
 import io.gatling.core.session.Expression
 import io.gatling.core.structure.ScenarioContext
 import io.gatling.http.request.builder.{Http, HttpRequestBuilder}
+import spray.json.DefaultJsonProtocol._
+import spray.json._
 
 case class OpenWhiskActionBuilderBase(requestName: Expression[String]) {
 
   implicit private val http = new Http(requestName)
 
-  def info() = OpenWhiskActionBuilder(http.get("/api/v1"))
+  /** Call the `/api/v1`-endpoint of the specified system */
+  def info() = {
+    OpenWhiskActionBuilder(http.get("/api/v1"))
+  }
+
+  /**
+   * Specify authentication data. This is needed to perform operations on namespaces or working with entities.
+   *
+   * @param uuid The UUID of the namespace
+   * @param key The key of the namespace
+   */
+  def authenticate(uuid: Expression[String], key: Expression[String]) = {
+    OpenWhiskActionBuilderWithNamespace(uuid, key)
+  }
+}
+
+case class OpenWhiskActionBuilderWithNamespace(private val uuid: Expression[String],
+                                               private val key: Expression[String],
+                                               private val namespace: String = "_")(implicit private val http: Http) {
+
+  /**
+   * Specify on which namespace you want to perform any action.
+   *
+   * @param namespace The namespace you want to use.
+   */
+  def namespace(namespace: String) = {
+    OpenWhiskActionBuilderWithNamespace(uuid, key, namespace)
+  }
+
+  /** List all namespaces you have access to, with your current authentication. */
+  def list() = {
+    OpenWhiskActionBuilder(http.get("/api/v1/namespaces").basicAuth(uuid, key))
+  }
+
+  /**
+   * Perform any request against the actions-API. E.g. creating, invoking or deleting actions.
+   *
+   * @param actionName Name of the action in the Whisk-system.
+   */
+  def action(actionName: String) = {
+    OpenWhiskActionBuilderWithAction(uuid, key, namespace, actionName)
+  }
+}
+
+case class OpenWhiskActionBuilderWithAction(private val uuid: Expression[String],
+                                            private val key: Expression[String],
+                                            private val namespace: String,
+                                            private val action: String)(implicit private val http: Http) {
+  private val path: Expression[String] = s"/api/v1/namespaces/$namespace/actions/$action"
+
+  /** Fetch the action from OpenWhisk */
+  def get() = {
+    OpenWhiskActionBuilder(http.get(path).basicAuth(uuid, key))
+  }
+
+  /** Delete the action from OpenWhisk */
+  def delete() = {
+    OpenWhiskActionBuilder(http.delete(path).basicAuth(uuid, key))
+  }
+
+  /**
+   * Create the action in OpenWhisk.
+   *
+   * @param code The code of the action to create.
+   * @param kind The kind of the action you want to create. Default is `nodejs:default`.
+   * @param main Main method of your action. This is only needed for java actions.
+   */
+  def create(code: Expression[String], kind: Expression[String] = "nodejs:default", main: Expression[String] = "") = {
+
+    val json: Expression[String] = session => {
+      code(session).flatMap { c =>
+        kind(session).flatMap { k =>
+          main(session).map { m =>
+            val exec = Map("kind" -> k, "code" -> c) ++ (if (m.size > 0) Map("main" -> m) else Map[String, String]())
+            JsObject("exec" -> exec.toJson).compactPrint
+          }
+        }
+      }
+    }
+
+    OpenWhiskActionBuilder(http.put(path).basicAuth(uuid, key).body(StringBody(json)))
+  }
+
+  /** Invoke the action. */
+  def invoke() = {
+    OpenWhiskActionBuilder(http.post(path).queryParam("blocking", "true").basicAuth(uuid, key))
+  }
 }
 
 case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder {


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services