You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ma...@apache.org on 2018/08/08 14:12:07 UTC
[incubator-openwhisk] branch master updated: Make error count and
percentage configurable for gatling tests. (#3942)
This is an automated email from the ASF dual-hosted git repository.
markusthoemmes pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new 92a4a0c Make error count and percentage configurable for gatling tests. (#3942)
92a4a0c is described below
commit 92a4a0c314821681043716cc601b64d543f61e80
Author: Jeremias Werner <je...@gmail.com>
AuthorDate: Wed Aug 8 16:12:01 2018 +0200
Make error count and percentage configurable for gatling tests. (#3942)
---
tests/performance/README.md | 56 +++++++++++++---------
.../src/gatling/scala/ApiV1Simulation.scala | 6 ++-
.../scala/BlockingInvokeOneActionSimulation.scala | 6 ++-
.../scala/ColdBlockingInvokeSimulation.scala | 6 ++-
.../src/gatling/scala/LatencySimulation.scala | 37 ++++++++++----
5 files changed, 75 insertions(+), 36 deletions(-)
diff --git a/tests/performance/README.md b/tests/performance/README.md
index f09d91b..dd780c3 100644
--- a/tests/performance/README.md
+++ b/tests/performance/README.md
@@ -70,11 +70,13 @@ The test is doing as many requests as possible for the given amount of time (`SE
Available environment variables:
```
-OPENWHISK_HOST (required)
-CONNECTIONS (required)
-SECONDS (default: 10)
-REQUESTS_PER_SEC (required)
-MIN_REQUESTS_PER_SEC (default: REQUESTS_PER_SEC)
+OPENWHISK_HOST (required)
+CONNECTIONS (required)
+SECONDS (default: 10)
+REQUESTS_PER_SEC (required)
+MIN_REQUESTS_PER_SEC (default: REQUESTS_PER_SEC)
+MAX_ERRORS_ALLOWED (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
```
You can run the simulation with (in OPENWHISK_HOME)
@@ -101,14 +103,20 @@ The comparison of the thresholds is against the mean response times of the warm
Available environment variables:
```
-OPENWHISK_HOST (required)
-API_KEY (required, format: UUID:KEY)
-PAUSE_BETWEEN_INVOKES (default: 0)
-MEAN_RESPONSE_TIME (required)
-MAX_MEAN_RESPONSE_TIME (default: MEAN_RESPONSE_TIME)
-EXCLUDED_KINDS (default: "", format: "python:default,java:default,swift:default")
+OPENWHISK_HOST (required)
+API_KEY (required, format: UUID:KEY)
+PAUSE_BETWEEN_INVOKES (default: 0)
+MEAN_RESPONSE_TIME (required)
+MAX_MEAN_RESPONSE_TIME (default: MEAN_RESPONSE_TIME)
+EXCLUDED_KINDS (default: "", format: "python:default,java:default,swift:default")
+MAX_ERRORS_ALLOWED (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
```
+It is possible to override the `MEAN_RESPONSE_TIME`, `MAX_MEAN_RESPONSE_TIME`, `MAX_ERRORS_ALLOWED` and `MAX_ERRORS_ALLOWED_PERCENTAGE`
+for each kind by adding the kind as prefix in upper case, like `JAVA_MEAN_RESPONSE_TIME`.
+
+
You can run the simulation with (in OPENWHISK_HOME)
```
OPENWHISK_HOST="openwhisk.mydomain.com" MEAN_RESPONSE_TIME="20" API_KEY="UUID:KEY" ./gradlew gatlingRun-LatencySimulation
@@ -130,12 +138,14 @@ The test is doing as many requests as possible for the given amount of time (`SE
Available environment variables:
```
-OPENWHISK_HOST (required)
-API_KEY (required, format: UUID:KEY)
-CONNECTIONS (required)
-SECONDS (default: 10)
-REQUESTS_PER_SEC (required)
-MIN_REQUESTS_PER_SEC (default: REQUESTS_PER_SEC)
+OPENWHISK_HOST (required)
+API_KEY (required, format: UUID:KEY)
+CONNECTIONS (required)
+SECONDS (default: 10)
+REQUESTS_PER_SEC (required)
+MIN_REQUESTS_PER_SEC (default: REQUESTS_PER_SEC)
+MAX_ERRORS_ALLOWED (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
```
You can run the simulation with
@@ -160,11 +170,13 @@ The test is doing as many requests as possible for the given amount of time (`SE
Available environment variables:
```
-OPENWHISK_HOST (required)
-USERS (required)
-SECONDS (default: 10)
-REQUESTS_PER_SEC (required)
-MIN_REQUESTS_PER_SEC (default: REQUESTS_PER_SEC)
+OPENWHISK_HOST (required)
+USERS (required)
+SECONDS (default: 10)
+REQUESTS_PER_SEC (required)
+MIN_REQUESTS_PER_SEC (default: REQUESTS_PER_SEC)
+MAX_ERRORS_ALLOWED (default: 0)
+MAX_ERRORS_ALLOWED_PERCENTAGE (default: 0)
```
You can run the simulation with
diff --git a/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala b/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala
index 91c5302..9fa7604 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/ApiV1Simulation.scala
@@ -30,6 +30,8 @@ class ApiV1Simulation extends Simulation {
// Specify thresholds
val requestsPerSec = sys.env("REQUESTS_PER_SEC").toInt
val minimalRequestsPerSec = sys.env.getOrElse("MIN_REQUESTS_PER_SEC", requestsPerSec.toString).toInt
+ val maxErrorsAllowed: Int = sys.env.getOrElse("MAX_ERRORS_ALLOWED", "0").toInt
+ val maxErrorsAllowedPercentage: Double = sys.env.getOrElse("MAX_ERRORS_ALLOWED_PERCENTAGE", "0.1").toDouble
// Generate the OpenWhiskProtocol
val openWhiskProtocol = openWhisk.apiHost(host)
@@ -46,6 +48,6 @@ class ApiV1Simulation extends Simulation {
.assertions(details("Call api/v1 endpoint").requestsPerSec.gt(minimalRequestsPerSec))
.assertions(details("Call api/v1 endpoint").requestsPerSec.gt(requestsPerSec))
// Mark the build yellow, if there are failed requests. And red if both conditions fail.
- .assertions(details("Call api/v1 endpoint").failedRequests.count.is(0))
- .assertions(details("Call api/v1 endpoint").failedRequests.percent.lte(0.1))
+ .assertions(details("Call api/v1 endpoint").failedRequests.count.lte(maxErrorsAllowed))
+ .assertions(details("Call api/v1 endpoint").failedRequests.percent.lte(maxErrorsAllowedPercentage))
}
diff --git a/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala b/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
index 79f2751..e49ce52 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/BlockingInvokeOneActionSimulation.scala
@@ -39,6 +39,8 @@ class BlockingInvokeOneActionSimulation extends Simulation {
// Specify thresholds
val requestsPerSec: Int = sys.env("REQUESTS_PER_SEC").toInt
val minimalRequestsPerSec: Int = sys.env.getOrElse("MIN_REQUESTS_PER_SEC", requestsPerSec.toString).toInt
+ val maxErrorsAllowed: Int = sys.env.getOrElse("MAX_ERRORS_ALLOWED", "0").toInt
+ val maxErrorsAllowedPercentage: Double = sys.env.getOrElse("MAX_ERRORS_ALLOWED_PERCENTAGE", "0.1").toDouble
// Generate the OpenWhiskProtocol
val openWhiskProtocol: OpenWhiskProtocolBuilder = openWhisk.apiHost(host)
@@ -74,6 +76,6 @@ class BlockingInvokeOneActionSimulation extends Simulation {
.assertions(details("Invoke action").requestsPerSec.gt(minimalRequestsPerSec))
.assertions(details("Invoke action").requestsPerSec.gt(requestsPerSec))
// Mark the build yellow, if there are failed requests. And red if both conditions fail.
- .assertions(details("Invoke action").failedRequests.count.is(0))
- .assertions(details("Invoke action").failedRequests.percent.lte(0.1))
+ .assertions(details("Invoke action").failedRequests.count.lte(maxErrorsAllowed))
+ .assertions(details("Invoke action").failedRequests.percent.lte(maxErrorsAllowedPercentage))
}
diff --git a/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala b/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
index 1e06a8b..d7ec74f 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/ColdBlockingInvokeSimulation.scala
@@ -38,6 +38,8 @@ class ColdBlockingInvokeSimulation extends Simulation {
// Specify thresholds
val requestsPerSec: Int = sys.env("REQUESTS_PER_SEC").toInt
val minimalRequestsPerSec: Int = sys.env.getOrElse("MIN_REQUESTS_PER_SEC", requestsPerSec.toString).toInt
+ val maxErrorsAllowed: Int = sys.env.getOrElse("MAX_ERRORS_ALLOWED", "0").toInt
+ val maxErrorsAllowedPercentage: Double = sys.env.getOrElse("MAX_ERRORS_ALLOWED_PERCENTAGE", "0.1").toDouble
// Generate the OpenWhiskProtocol
val openWhiskProtocol: OpenWhiskProtocolBuilder = openWhisk.apiHost(host)
@@ -85,6 +87,6 @@ class ColdBlockingInvokeSimulation extends Simulation {
.assertions(details("Invoke action").requestsPerSec.gt(minimalRequestsPerSec))
.assertions(details("Invoke action").requestsPerSec.gt(requestsPerSec))
// Mark the build yellow, if there are failed requests. And red if both conditions fail.
- .assertions(details("Invoke action").failedRequests.count.is(0))
- .assertions(details("Invoke action").failedRequests.percent.lte(0.1))
+ .assertions(details("Invoke action").failedRequests.count.lte(maxErrorsAllowed))
+ .assertions(details("Invoke action").failedRequests.percent.lte(maxErrorsAllowedPercentage))
}
diff --git a/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala b/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
index a685681..8830c78 100644
--- a/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
+++ b/tests/performance/gatling_tests/src/gatling/scala/LatencySimulation.scala
@@ -35,9 +35,19 @@ class LatencySimulation extends Simulation {
val pauseBetweenInvokes: Int = sys.env.getOrElse("PAUSE_BETWEEN_INVOKES", "0").toInt
+ val MEAN_RESPONSE_TIME = "MEAN_RESPONSE_TIME"
+ val MAX_MEAN_RESPONSE_TIME = "MAX_MEAN_RESPONSE_TIME"
+ val MAX_ERRORS_ALLOWED = "MAX_ERRORS_ALLOWED"
+ val MAX_ERRORS_ALLOWED_PERCENTAGE = "MAX_ERRORS_ALLOWED_PERCENTAGE"
+
// Specify thresholds
- val meanResponseTime: Int = sys.env("MEAN_RESPONSE_TIME").toInt
- val maximalMeanResponseTime: Int = sys.env.getOrElse("MAX_MEAN_RESPONSE_TIME", meanResponseTime.toString).toInt
+ val meanResponseTime: Int = sys.env(MEAN_RESPONSE_TIME).toInt
+ val maximalMeanResponseTime: Int = sys.env.getOrElse(MAX_MEAN_RESPONSE_TIME, meanResponseTime.toString).toInt
+
+ val maxErrorsAllowed: Int = sys.env.getOrElse(MAX_ERRORS_ALLOWED, "0").toInt
+ val maxErrorsAllowedPercentage: Double = sys.env.getOrElse(MAX_ERRORS_ALLOWED_PERCENTAGE, "0.1").toDouble
+
+ def toKindSpecificKey(kind: String, suffix: String) = kind.split(':').head.toUpperCase + "_" + suffix
// Exclude runtimes
val excludedKinds: Seq[String] = sys.env.getOrElse("EXCLUDED_KINDS", "").split(",")
@@ -91,14 +101,25 @@ class LatencySimulation extends Simulation {
.protocols(openWhiskProtocol)
actions
- .map { case (kind, _, _, _) => s"Warm $kind invocation" }
- .foldLeft(testSetup) { (agg, cur) =>
+ .map { case (kind, _, _, _) => kind }
+ .foldLeft(testSetup) { (agg, kind) =>
+ val cur = s"Warm $kind invocation"
// One failure will make the build yellow
+ val specificMeanResponseTime: Int =
+ sys.env.getOrElse(toKindSpecificKey(kind, MEAN_RESPONSE_TIME), meanResponseTime.toString).toInt
+ val specificMaxMeanResponseTime =
+ sys.env.getOrElse(toKindSpecificKey(kind, MAX_MEAN_RESPONSE_TIME), maximalMeanResponseTime.toString).toInt
+ val specificMaxErrorsAllowed =
+ sys.env.getOrElse(toKindSpecificKey(kind, MAX_ERRORS_ALLOWED), maxErrorsAllowed.toString).toInt
+ val specificMaxErrorsAllowedPercentage = sys.env
+ .getOrElse(toKindSpecificKey(kind, MAX_ERRORS_ALLOWED_PERCENTAGE), maxErrorsAllowedPercentage.toString)
+ .toDouble
+
agg
- .assertions(details(cur).responseTime.mean.lte(meanResponseTime))
- .assertions(details(cur).responseTime.mean.lt(maximalMeanResponseTime))
+ .assertions(details(cur).responseTime.mean.lte(specificMeanResponseTime))
+ .assertions(details(cur).responseTime.mean.lt(specificMaxMeanResponseTime))
// Mark the build yellow, if there are failed requests. And red if both conditions fail.
- .assertions(details(cur).failedRequests.count.is(0))
- .assertions(details(cur).failedRequests.percent.lte(0.1))
+ .assertions(details(cur).failedRequests.count.lte(specificMaxErrorsAllowed))
+ .assertions(details(cur).failedRequests.percent.lte(specificMaxErrorsAllowedPercentage))
}
}