You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ma...@apache.org on 2018/04/18 14:06:01 UTC
[incubator-openwhisk] branch master updated: Add gatling-based
latency test. (#3556)
This is an automated email from the ASF dual-hosted git repository.
markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 866990e Add gatling-based latency test. (#3556)
866990e is described below
commit 866990eefc1418e1a8775b05fca34031a6b096b6
Author: Christian Bickel <gi...@cbickel.de>
AuthorDate: Wed Apr 18 16:05:57 2018 +0200
Add gatling-based latency test. (#3556)
---
.travis.yml | 1 +
performance/README.md | 30 +++++++
performance/gatling_tests/build.gradle | 6 +-
.../src/gatling/resources/data/javaAction.jar | Bin 0 -> 1060 bytes
.../data/javaAction.java} | 34 ++++----
.../src/gatling/resources/data/nodeJSAction.js | 5 ++
.../src/gatling/resources/data/pythonAction.py | 9 ++
.../src/gatling/resources/data/swiftAction.swift | 9 ++
.../src/gatling/scala/LatencySimulation.scala | 97 +++++++++++++++++++++
.../extension/whisk/OpenWhiskActionBuilder.scala | 90 ++++++++++++++++++-
10 files changed, 261 insertions(+), 20 deletions(-)
diff --git a/.travis.yml b/.travis.yml
index 7fc1587..f5b0fd8 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -65,5 +65,6 @@ jobs:
- TERM=dumb ./performance/wrk_tests/latency.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" 2m
- TERM=dumb ./performance/wrk_tests/throughput.sh "https://172.17.0.1:10001" "$(cat ansible/files/auth.guest)" 4 2 2m
- OPENWHISK_HOST="172.17.0.1" CONNECTIONS="100" REQUESTS_PER_SEC="1" ./gradlew gatlingRun-ApiV1Simulation
+ - OPENWHISK_HOST="172.17.0.1" MEAN_RESPONSE_TIME="1000" API_KEY="$(cat ansible/files/auth.guest)" EXCLUDED_KINDS="python:default,java:default,swift:default" ./gradlew gatlingRun-LatencySimulation
env:
- DESCRIPTION="Execute wrk-performance test suite."
diff --git a/performance/README.md b/performance/README.md
index f089021..e8d2164 100644
--- a/performance/README.md
+++ b/performance/README.md
@@ -61,3 +61,33 @@ You can run the simulation with (in OPENWHISK_HOME)
```
OPENWHISK_HOST="openwhisk.mydomain.com" CONNECTIONS="10" REQUESTS_PER_SEC="50" ./gradlew gatlingRun-ApiV1Simulation
```
+
+##### Latency Simulation
+
+This simulation creates actions of the following four kinds: `nodejs:default`, `swift:default`, `java:default` and
+`python:default`.
+Afterwards the action is invoked once. This is the cold-start and will not be part of the thresholds.
+Next, the action will be invoked 100 times blocking and one after each other. The last step is, that the action will be deleted.
+
+Once one language is finished, the next kind will be taken. They are not running in parallel. There are never more than
+1 activations in the system, as we only want to meassure latency of warm activations.
+As all actions are invoked blocking and only one action is in the system, it doesn't matter how many controllers
+and invokers are deployed. If several controllers or invokers are deployed, all controllers send the activation
+always to the same invoker.
+
+The comparison of the thresholds is against the mean response times of the warm activations.
+
+Available environment variables:
+
+```
+OPENWHISK_HOST (required)
+API_KEY (required, format: UUID:KEY)
+MEAN_RESPONSE_TIME (required)
+MAX_MEAN_RESPONSE_TIME (default: MEAN_RESPONSE_TIME)
+EXCLUDED_KINDS (default: "", format: "python:default,java:default,swift:default")
+```
+
+You can run the simulation with (in OPENWHISK_HOME)
+```
+OPENWHISK_HOST="openwhisk.mydomain.com" MEAN_RESPONSE_TIME="20" API_KEY="UUID:KEY" ./gradlew gatlingRun-LatencySimulation
+```
diff --git a/performance/gatling_tests/build.gradle b/performance/gatling_tests/build.gradle
index 7e8f1ad..9242fb4 100644
--- a/performance/gatling_tests/build.gradle
+++ b/performance/gatling_tests/build.gradle
@@ -1,5 +1,5 @@
plugins {
- id "com.github.lkishalmi.gatling" version "0.7.1"
+ id "com.github.lkishalmi.gatling" version "0.7.2"
}
apply plugin: 'eclipse'
@@ -10,8 +10,8 @@ repositories {
}
dependencies {
- compile "org.scala-lang:scala-library:${gradle.scala.version}"
- compile "io.gatling.highcharts:gatling-charts-highcharts:2.2.5"
+ gatling "io.spray:spray-json_2.12:1.3.4"
+ gatling "commons-io:commons-io:2.6"
}
tasks.withType(ScalaCompile) {
diff --git a/performance/gatling_tests/src/gatling/resources/data/javaAction.jar b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar
new file mode 100644
index 0000000..17229eb
Binary files /dev/null and b/performance/gatling_tests/src/gatling/resources/data/javaAction.jar differ
diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala b/performance/gatling_tests/src/gatling/resources/data/javaAction.java
similarity index 54%
copy from performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
copy to performance/gatling_tests/src/gatling/resources/data/javaAction.java
index ef6d2ef..56da8db 100644
--- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
+++ b/performance/gatling_tests/src/gatling/resources/data/javaAction.java
@@ -15,24 +15,26 @@
* limitations under the License.
*/
-package extension.whisk
+// Build the jar with the following commands:
+//
+// javac -cp gson-2.8.2.jar JavaAction.java
+// jar cvf javaAction.jar JavaAction.class
-import io.gatling.core.Predef._
-import io.gatling.core.action.Action
-import io.gatling.core.action.builder.ActionBuilder
-import io.gatling.core.session.Expression
-import io.gatling.core.structure.ScenarioContext
-import io.gatling.http.request.builder.{Http, HttpRequestBuilder}
+import com.google.gson.JsonObject;
-case class OpenWhiskActionBuilderBase(requestName: Expression[String]) {
+public class JavaAction {
+ public static JsonObject main(JsonObject args) {
+ String text;
- implicit private val http = new Http(requestName)
+ try {
+ text = args.getAsJsonPrimitive("text").getAsString();
+ } catch(Exception e) {
+ text = "stranger";
+ }
- def info() = OpenWhiskActionBuilder(http.get("/api/v1"))
-}
-
-case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder {
- override def build(ctx: ScenarioContext, next: Action): Action = {
- http.build(ctx, next)
- }
+ JsonObject response = new JsonObject();
+ System.out.println("Hello " + text + "!");
+ response.addProperty("payload", "Hello " + text + "!");
+ return response;
+ }
}
diff --git a/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js
new file mode 100644
index 0000000..8c71017
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/nodeJSAction.js
@@ -0,0 +1,5 @@
+function main(params) {
+ var greeting = "Hello" + (params.text || "stranger") + "!";
+ console.log(greeting);
+ return { payload: greeting };
+}
diff --git a/performance/gatling_tests/src/gatling/resources/data/pythonAction.py b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py
new file mode 100644
index 0000000..c26b060
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/pythonAction.py
@@ -0,0 +1,9 @@
+import sys
+def main(dict):
+ if 'text' in dict:
+ text = dict['text']
+ else:
+ text = "stranger"
+ greeting = "Hello " + text + "!"
+ print(greeting)
+ return {"payload": greeting}
diff --git a/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift
new file mode 100644
index 0000000..3f52f66
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/resources/data/swiftAction.swift
@@ -0,0 +1,9 @@
+func main(args: [String:Any]) -> [String:Any] {
+ if let text = args["text"] as? String {
+ print("Hello " + text + "!")
+ return [ "payload" : "Hello " + text + "!" ]
+ } else {
+ print("Hello stranger!")
+ return [ "payload" : "Hello stranger!" ]
+ }
+}
diff --git a/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
new file mode 100644
index 0000000..9ec41f0
--- /dev/null
+++ b/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.nio.charset.StandardCharsets
+import java.util.Base64
+
+import extension.whisk.Predef._
+import io.gatling.core.Predef._
+import io.gatling.core.session.Expression
+import io.gatling.core.util.Resource
+import org.apache.commons.io.FileUtils
+
+class LatencySimulation extends Simulation {
+ // Specify parameters for the run
+ val host = sys.env("OPENWHISK_HOST")
+
+ // Specify authentication
+ val Array(uuid, key) = sys.env("API_KEY").split(":")
+
+ // Specify thresholds
+ val meanResponseTime: Int = sys.env("MEAN_RESPONSE_TIME").toInt
+ val maximalMeanResponseTime: Int = sys.env.getOrElse("MAX_MEAN_RESPONSE_TIME", meanResponseTime.toString).toInt
+
+ // Exclude runtimes
+ val excludedKinds: Seq[String] = sys.env.getOrElse("EXCLUDED_KINDS", "").split(",")
+
+ // Generate the OpenWhiskProtocol
+ val openWhiskProtocol = openWhisk.apiHost(host)
+
+ /**
+ * Generate a list of actions to execute. The list is a tuple of (kind, code, actionName, main)
+ * `kind` is needed to create the action
+ * `code` is loaded form the files located in `resources/data`
+ * `actionName` is the name of the action in OpenWhisk
+ * `main` is only needed for java. This is the name of the class where the main method is located.
+ */
+ val actions: Seq[(String, String, String, String)] = Map(
+ "nodejs:default" -> (FileUtils
+ .readFileToString(Resource.body("nodeJSAction.js").get.file, StandardCharsets.UTF_8), "latencyTest_node", ""),
+ "python:default" -> (FileUtils
+ .readFileToString(Resource.body("pythonAction.py").get.file, StandardCharsets.UTF_8), "latencyTest_python", ""),
+ "swift:default" -> (FileUtils
+ .readFileToString(Resource.body("swiftAction.swift").get.file, StandardCharsets.UTF_8), "latencyTest_swift", ""),
+ "java:default" -> (Base64.getEncoder.encodeToString(
+ FileUtils.readFileToByteArray(Resource.body("javaAction.jar").get.file)), "latencyTest_java", "JavaAction"))
+ .filterNot(e => excludedKinds.contains(e._1))
+ .map {
+ case (kind, (code, name, main)) =>
+ (kind, code, name, main)
+ }
+ .toSeq
+
+ // Define scenario
+ val test = scenario("Invoke one action after each other to test latency")
+ .foreach(actions, "action") {
+ val code: Expression[String] = "${action._2}"
+ exec(
+ openWhisk("Create ${action._1} action")
+ .authenticate(uuid, key)
+ .action("${action._3}")
+ .create(code, "${action._1}", "${action._4}"))
+ .exec(openWhisk("Cold ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke())
+ .repeat(100) {
+ exec(openWhisk("Warm ${action._1} invocation").authenticate(uuid, key).action("${action._3}").invoke())
+ }
+ .exec(openWhisk("Delete ${action._1} action").authenticate(uuid, key).action("${action._3}").delete())
+ }
+
+ val testSetup = setUp(test.inject(atOnceUsers(1)))
+ .protocols(openWhiskProtocol)
+
+ actions
+ .map { case (kind, _, _, _) => s"Warm $kind invocation" }
+ .foldLeft(testSetup) { (agg, cur) =>
+ // One failure will make the build yellow
+ agg
+ .assertions(details(cur).responseTime.mean.lte(meanResponseTime))
+ .assertions(details(cur).responseTime.mean.lt(maximalMeanResponseTime))
+ // Mark the build yellow, if there are failed requests. And red if both conditions fail.
+ .assertions(details(cur).failedRequests.count.is(0))
+ .assertions(details(cur).failedRequests.percent.lte(0.1))
+ }
+}
diff --git a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
index ef6d2ef..56dc6ce 100644
--- a/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
+++ b/performance/gatling_tests/src/gatling/scala/extension/whisk/OpenWhiskActionBuilder.scala
@@ -23,12 +23,100 @@ import io.gatling.core.action.builder.ActionBuilder
import io.gatling.core.session.Expression
import io.gatling.core.structure.ScenarioContext
import io.gatling.http.request.builder.{Http, HttpRequestBuilder}
+import spray.json.DefaultJsonProtocol._
+import spray.json._
case class OpenWhiskActionBuilderBase(requestName: Expression[String]) {
implicit private val http = new Http(requestName)
- def info() = OpenWhiskActionBuilder(http.get("/api/v1"))
+ /** Call the `/api/v1`-endpoint of the specified system */
+ def info() = {
+ OpenWhiskActionBuilder(http.get("/api/v1"))
+ }
+
+ /**
+ * Specify authentication data. This is needed to perform operations on namespaces or working with entities.
+ *
+ * @param uuid The UUID of the namespace
+ * @param key The key of the namespace
+ */
+ def authenticate(uuid: Expression[String], key: Expression[String]) = {
+ OpenWhiskActionBuilderWithNamespace(uuid, key)
+ }
+}
+
+case class OpenWhiskActionBuilderWithNamespace(private val uuid: Expression[String],
+ private val key: Expression[String],
+ private val namespace: String = "_")(implicit private val http: Http) {
+
+ /**
+ * Specify on which namespace you want to perform any action.
+ *
+ * @param namespace The namespace you want to use.
+ */
+ def namespace(namespace: String) = {
+ OpenWhiskActionBuilderWithNamespace(uuid, key, namespace)
+ }
+
+ /** List all namespaces you have access to, with your current authentication. */
+ def list() = {
+ OpenWhiskActionBuilder(http.get("/api/v1/namespaces").basicAuth(uuid, key))
+ }
+
+ /**
+ * Perform any request against the actions-API. E.g. creating, invoking or deleting actions.
+ *
+ * @param actionName Name of the action in the Whisk-system.
+ */
+ def action(actionName: String) = {
+ OpenWhiskActionBuilderWithAction(uuid, key, namespace, actionName)
+ }
+}
+
+case class OpenWhiskActionBuilderWithAction(private val uuid: Expression[String],
+ private val key: Expression[String],
+ private val namespace: String,
+ private val action: String)(implicit private val http: Http) {
+ private val path: Expression[String] = s"/api/v1/namespaces/$namespace/actions/$action"
+
+ /** Fetch the action from OpenWhisk */
+ def get() = {
+ OpenWhiskActionBuilder(http.get(path).basicAuth(uuid, key))
+ }
+
+ /** Delete the action from OpenWhisk */
+ def delete() = {
+ OpenWhiskActionBuilder(http.delete(path).basicAuth(uuid, key))
+ }
+
+ /**
+ * Create the action in OpenWhisk.
+ *
+ * @param code The code of the action to create.
+ * @param kind The kind of the action you want to create. Default is `nodejs:default`.
+ * @param main Main method of your action. This is only needed for java actions.
+ */
+ def create(code: Expression[String], kind: Expression[String] = "nodejs:default", main: Expression[String] = "") = {
+
+ val json: Expression[String] = session => {
+ code(session).flatMap { c =>
+ kind(session).flatMap { k =>
+ main(session).map { m =>
+ val exec = Map("kind" -> k, "code" -> c) ++ (if (m.size > 0) Map("main" -> m) else Map[String, String]())
+ JsObject("exec" -> exec.toJson).compactPrint
+ }
+ }
+ }
+ }
+
+ OpenWhiskActionBuilder(http.put(path).basicAuth(uuid, key).body(StringBody(json)))
+ }
+
+ /** Invoke the action. */
+ def invoke() = {
+ OpenWhiskActionBuilder(http.post(path).queryParam("blocking", "true").basicAuth(uuid, key))
+ }
}
case class OpenWhiskActionBuilder(http: HttpRequestBuilder) extends ActionBuilder {
--
To stop receiving notification emails like this one, please contact
markusthoemmes@apache.org.