You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2015/11/02 18:17:08 UTC
[01/17] usergrid git commit: USERGRID-1044: set default retry count
to 0 also, retry count was being used as the total number of attempts
Repository: usergrid
Updated Branches:
refs/heads/master b031d4d3d -> 568d35d57
USERGRID-1044: set default retry count to 0
also, retry count was being used as the total number of attempts
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1195ce85
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1195ce85
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1195ce85
Branch: refs/heads/master
Commit: 1195ce85fefcc8334b38f6e79cc8a2c53ce9ce10
Parents: 2f307a7
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 20 13:37:34 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 20 13:37:34 2015 -0700
----------------------------------------------------------------------
stack/loadtests/README.md | 2 +-
.../org/apache/usergrid/enums/ConfigProperties.scala | 2 +-
.../apache/usergrid/scenarios/AuditScenarios.scala | 6 +++---
.../scenarios/EntityCollectionScenarios.scala | 14 +++++++-------
stack/loadtests/testConfig.sh | 4 ++--
5 files changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1195ce85/stack/loadtests/README.md
----------------------------------------------------------------------
diff --git a/stack/loadtests/README.md b/stack/loadtests/README.md
index 802c286..ec34b42 100644
--- a/stack/loadtests/README.md
+++ b/stack/loadtests/README.md
@@ -106,7 +106,7 @@ Defaults listed are those that are specified by the Usergrid Gatling code, not n
* uuidFilename (no default) - UUID filename for non-audit tests
* auditUuidFilename (no default) - UUID filename for audit tests
* failedUuidFilename (no default) - UUID filename for failed entries while auditing
-* retryCount (**5**) - number of retries of operation before giving up
+* retryCount (**0**) - number of retries of operation before giving up
* purgeUsers (**100**) - number of entities to create at a time during loadEntities (send this many requests, then wait for the responses)
* laterThanTimestamp (**0**) - if specified for an audit, will only match entities modified >= this timestamp; can be used for incremental audits
* entityProgressCount (**10000**) - print to console every time this many requests has been sent (if entityProgressCount = 10000, will print on the 1000th, 2000th, 3000th... request)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1195ce85/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index f74222e..cbb484c 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -153,7 +153,7 @@ object ConfigProperties {
case FailedUuidFilename => "/tmp/dummyFailedUuid.csv"
case SandboxCollection => false
case PurgeUsers => 100
- case RetryCount => 5
+ case RetryCount => 0
case LaterThanTimestamp => 0L
case EntityProgressCount => 10000L
case InjectionList => "rampUsers(10,60)"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1195ce85/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 4743536..4477338 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -100,10 +100,10 @@ object AuditScenarios {
session
}
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
exec(getCollectionsWithoutCursor)
}.asLongAs(stringParamExists(SessionVarCursor)) {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
exec(getCollectionsWithCursor)
}
}
@@ -252,7 +252,7 @@ object AuditScenarios {
.asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
feed(FeederGenerator.collectionCsvFeeder)
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
exec(getCollectionEntity)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1195ce85/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index f717f16..b56917a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -129,7 +129,7 @@ object EntityCollectionScenarios {
.exec(injectAuthType())
.doIfOrElse(_ => Settings.endConditionType == EndConditionType.MinutesElapsed) {
asLongAs(_ => Settings.continueMinutesTest) {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getRandomEntityAnonymous)
} {
@@ -139,7 +139,7 @@ object EntityCollectionScenarios {
}
} {
repeat(_ => Settings.endRequestCount.toInt) {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getRandomEntityAnonymous)
} {
@@ -180,7 +180,7 @@ object EntityCollectionScenarios {
session => println(s"UUID: ${session("uuid").as[String]}")
session
}*/
- .tryMax(Settings.retryCount) {
+ .tryMax(1+Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getRandomEntityByUuidAnonymous)
} {
@@ -195,7 +195,7 @@ object EntityCollectionScenarios {
session => println(s"UUID: ${session("uuid").as[String]}")
session
}*/
- .tryMax(Settings.retryCount) {
+ .tryMax(1+Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getRandomEntityByUuidAnonymous)
} {
@@ -238,7 +238,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
exec(loadEntity)
}
}
@@ -266,7 +266,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
exec(deleteEntity)
}
}
@@ -350,7 +350,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(Settings.retryCount) {
+ tryMax(1+Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getEntityByNameSequentialAnonymous)
} {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1195ce85/stack/loadtests/testConfig.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/testConfig.sh b/stack/loadtests/testConfig.sh
index 59f64e1..fc4de12 100755
--- a/stack/loadtests/testConfig.sh
+++ b/stack/loadtests/testConfig.sh
@@ -40,11 +40,11 @@ SKIP_SETUP=false
COLLECTION=thousands
ENTITY_TYPE=trivialSortable
ENTITY_PREFIX=sortable
-ENTITY_SEED=1
+ENTITY_SEED=0
SEARCH_QUERY=order%20by%20sortField%20desc
SEARCH_LIMIT=1000
-RETRY_COUNT=5
+RETRY_COUNT=0
LATER_THAN_TIMESTAMP=0
ENTITY_PROGRESS_COUNT=1000
[05/17] usergrid git commit: USERGRID-1044: transform payload to
empty JSON if non-json
Posted by mr...@apache.org.
USERGRID-1044: transform payload to empty JSON if non-json
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2778f0a0
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2778f0a0
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2778f0a0
Branch: refs/heads/master
Commit: 2778f0a0b196bfc09e914033392ccb49271136e0
Parents: 3c60c49
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 10:49:44 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 10:49:44 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/helpers/Extractors.scala | 2 +-
.../scenarios/EntityCollectionScenarios.scala | 17 +++++++++++++++++
2 files changed, 18 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2778f0a0/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index b21f9d7..4c6ac8d 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -85,7 +85,7 @@ object Extractors {
def extractAuditEntity(saveAsName: String) = {
jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => {
extract.orElse(Some(Seq.empty))
- }).optional.saveAs(saveAsName)
+ }).saveAs(saveAsName)
}
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2778f0a0/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index ecd0c21..bcdb49a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -16,9 +16,13 @@
*/
package org.apache.usergrid.scenarios
+import java.nio.charset.StandardCharsets
+
import io.gatling.core.Predef._
import io.gatling.core.feeder.RecordSeqFeederBuilder
import io.gatling.http.Predef._
+import io.gatling.http.response.ResponseWrapper
+import io.gatling.http.response.StringResponseBody
import org.apache.usergrid.datagenerators.FeederGenerator
import org.apache.usergrid.enums.{CsvFeedPatternType, EndConditionType, AuthType}
import org.apache.usergrid.helpers.Extractors._
@@ -216,6 +220,19 @@ object EntityCollectionScenarios {
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.body(StringBody("""${entity}"""))
+ .transformResponse {
+ case response if response.isReceived =>
+ new ResponseWrapper(response) {
+ override val body = {
+ val contentType = response.header("content-type").getOrElse("").toLowerCase
+ if (contentType.contains("json")) {
+ StringResponseBody(response.body.string, response.charset)
+ } else {
+ StringResponseBody("{}", StandardCharsets.UTF_8)
+ }
+ }
+ }
+ }
// 200 for success, 400 if already exists
.check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
[14/17] usergrid git commit: USERGRID-1044: don't log UUID mismatch
if UUID was never retrieved during load
Posted by mr...@apache.org.
USERGRID-1044: don't log UUID mismatch if UUID was never retrieved during load
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0fc18041
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0fc18041
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0fc18041
Branch: refs/heads/master
Commit: 0fc18041e32df049040f26a0faaef1f46dbe6d9f
Parents: 2a09bf1
Author: Mike Dunker <md...@apigee.com>
Authored: Wed Oct 28 08:09:19 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Wed Oct 28 08:09:19 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0fc18041/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index d1e2c4b..cc59e0a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -183,7 +183,7 @@ object AuditScenarios {
errorString += "|NoUuidReturned"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISSING (DIRECT): requestedUuid=$uuid")
- } else if (!uuid.equalsIgnoreCase(entityUuid)) {
+ } else if (!uuid.isEmpty && !uuid.equalsIgnoreCase(entityUuid)) {
errorString += "|ReturnedUuidMismatch"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISMATCH (DIRECT): requestedUuid=$uuid returnedUuid=$entityUuid")
@@ -247,7 +247,7 @@ object AuditScenarios {
errorString += "|NoUuidReturned"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISSING (QUERY): requestedUuid=$uuid")
- } else if (!uuid.equalsIgnoreCase(entityUuid)) {
+ } else if (!uuid.isEmpty && !uuid.equalsIgnoreCase(entityUuid)) {
errorString += "|ReturnedUuidMismatch"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISMATCH (QUERY): requestedUuid=$uuid returnedUuid=$entityUuid")
[17/17] usergrid git commit: Merge commit 'refs/pull/421/head' of
github.com:apache/usergrid
Posted by mr...@apache.org.
Merge commit 'refs/pull/421/head' of github.com:apache/usergrid
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/568d35d5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/568d35d5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/568d35d5
Branch: refs/heads/master
Commit: 568d35d57e8a664a9eaf4ce9edf313d49e7e963b
Parents: b031d4d 73dcaeb
Author: Michael Russo <mi...@gmail.com>
Authored: Mon Nov 2 09:16:45 2015 -0800
Committer: Michael Russo <mi...@gmail.com>
Committed: Mon Nov 2 09:16:45 2015 -0800
----------------------------------------------------------------------
stack/loadtests/README.md | 14 +-
stack/loadtests/runAuditDeleteEntities.sh | 65 ++++++
.../runAuditGetAllAppCollectionEntities.sh | 6 +-
.../loadtests/runAuditGetCollectionEntities.sh | 6 +-
.../runAuditVerifyCollectionEntities.sh | 2 +-
stack/loadtests/runLoadEntitiesUnlimited.sh | 1 +
stack/loadtests/runOrgAppSetup.sh | 17 +-
.../datagenerators/FeederGenerator.scala | 16 +-
.../usergrid/enums/ConfigProperties.scala | 8 +-
.../apache/usergrid/enums/ScenarioType.scala | 3 +-
.../apache/usergrid/helpers/Extractors.scala | 18 +-
.../usergrid/scenarios/AuditScenarios.scala | 203 +++++++++++++------
.../scenarios/EntityCollectionScenarios.scala | 102 +++++++---
.../org/apache/usergrid/settings/Settings.scala | 42 ++--
.../usergrid/simulations/AuditSimulation.scala | 15 +-
.../simulations/ConfigurableSimulation.scala | 2 +-
stack/loadtests/testConfig.sh | 4 +-
17 files changed, 385 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
[04/17] usergrid git commit: USERGRID-1044: make jsonPath extraction
optional to keep users from exiting
Posted by mr...@apache.org.
USERGRID-1044: make jsonPath extraction optional to keep users from exiting
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3c60c49a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3c60c49a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3c60c49a
Branch: refs/heads/master
Commit: 3c60c49a157be63ec626ff88666472f2d4876554
Parents: 1a6de02
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 08:52:17 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 08:52:17 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/helpers/Extractors.scala | 18 +++++++++++-------
1 file changed, 11 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3c60c49a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index 136a561..b21f9d7 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -37,7 +37,7 @@ object Extractors {
jsonPath("$.cursor").transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as an empty string
extract.orElse(Some(""))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
@@ -47,7 +47,7 @@ object Extractors {
jsonPath("$.entities[0].uuid").transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as an empty string
extract.orElse(Some(""))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
@@ -57,7 +57,7 @@ object Extractors {
jsonPath("$.entities[0].name").transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as an empty string
extract.orElse(Some(""))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
@@ -67,21 +67,25 @@ object Extractors {
jsonPath("$.entities[0].modified").ofType[Long].transformOption(extract => {
//it may or may not be present. If it is, save it, otherwise save it as -1
extract.orElse(Some(-1))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
* Will extract the audit entities from the get collection response.
*/
def extractAuditEntities(saveAsName: String) = {
- jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+ jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract => {
+ extract.orElse(Some(Seq.empty))
+ }).optional.saveAs(saveAsName)
}
/**
* Will extract the audit entities from the get collection response.
*/
def extractAuditEntity(saveAsName: String) = {
- jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+ jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => {
+ extract.orElse(Some(Seq.empty))
+ }).optional.saveAs(saveAsName)
}
/**
@@ -100,7 +104,7 @@ object Extractors {
def maybeExtractEntities(saveAsName: String) = {
jsonPath("$.entities").ofType[Seq[Any]].transformOption(extract => {
extract.orElse(Some(Seq()))
- }).saveAs(saveAsName)
+ }).optional.saveAs(saveAsName)
}
/**
[09/17] usergrid git commit: USERGRID-1044: fixes
Posted by mr...@apache.org.
USERGRID-1044: fixes
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/83d7a6f8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/83d7a6f8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/83d7a6f8
Branch: refs/heads/master
Commit: 83d7a6f83e005ee8a4794ed86f4f0de95653360c
Parents: be0033e
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 13:44:33 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 13:44:33 2015 -0700
----------------------------------------------------------------------
.../usergrid/datagenerators/FeederGenerator.scala | 3 ++-
.../usergrid/scenarios/AuditScenarios.scala | 18 ++++++++----------
.../org/apache/usergrid/settings/Settings.scala | 10 +++++-----
3 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/83d7a6f8/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 73b3d3c..38db19a 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -279,11 +279,12 @@ object FeederGenerator {
val name = if (line != null) array(1) else ""
val uuid = if (line != null) array(2) else ""
val modified = if (line != null) array(3) else ""
+ val lastStatus = if (line != null) array(4) else ""
//println(s"$collectionName|$name|$uuid|$modified")
val accessField = if (uuid != "") uuid else name
val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
- Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified,
+ Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "lastStatus" -> lastStatus,
"validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/83d7a6f8/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 86c0b73..1ce2bc2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -157,8 +157,7 @@ object AuditScenarios {
val modified = session("modified").as[String].toLong
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
- val prevStatus = session("status").as[Int]
- val prevError = session("error").as[String]
+ val lastStatus = session("lastStatus").as[String]
if (status == 200 || status == 404) {
val collectionEntities = session(SessionVarCollectionEntities).as[Seq[Any]]
val entityUuid = session(SessionVarEntityUuid).as[String]
@@ -166,12 +165,12 @@ object AuditScenarios {
val count = collectionEntities.length
if (count < 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll", lastStatus)
Settings.incAuditNotFoundAtAll()
println(s"NOT FOUND AT ALL: $collectionName.$reqName ($uuid)")
} else if (count > 1) {
// invalid
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", lastStatus)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -199,13 +198,13 @@ object AuditScenarios {
println(s"PAYLOAD NAME MISMATCH (DIRECT): requestedName=$reqName returnedName=$entityName")
}
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, lastStatus)
Settings.incAuditNotFoundViaQuery()
println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
}
session
} else if (saveFailures) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", lastStatus)
session
} else {
session.markAsFailed
@@ -227,8 +226,7 @@ object AuditScenarios {
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
val modified = session("modified").as[String].toLong
- val prevStatus = session("status").as[Int]
- val prevError = session("error").as[String]
+ val lastStatus = session("lastStatus").as[String]
val collectionName = session(SessionVarCollectionName).as[String]
val entityUuid = session(SessionVarEntityUuid).as[String]
val entityName = session(SessionVarEntityName).as[String]
@@ -236,7 +234,7 @@ object AuditScenarios {
if (count < 1) {
// will check to see whether accessible directly
} else if (count > 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", lastStatus)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -268,7 +266,7 @@ object AuditScenarios {
// log even if technically successful -- we need to capture incorrect response
if (errorString != errorPrefix) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, lastStatus)
}
Settings.incAuditSuccess()
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/83d7a6f8/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index e0a0ec9..e27903a 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -291,10 +291,10 @@ object Settings {
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
val uuidsHeader = "collection,name,uuid,modified,status"
- val uuidsFailHeader = "collection,name,uuid,modified,status,error,prevStatus,prevError"
+ val uuidsFailHeader = "collection,name,uuid,modified,status,error,lastStatus"
case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int)
case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long,
- var status: Int, var error: String, var prevStatus: Int, var prevError: String)
+ var status: Int, var error: String, var lastStatus: String)
//private var uuidMap: Map[Int, String] = Map()
private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
@@ -383,10 +383,10 @@ object Settings {
private var lastAuditEntityCountPrinted: Long = 0L
private var auditUuidList: mutable.MutableList[AuditFailList] = mutable.MutableList[AuditFailList]()
def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String,
- prevStatus: Int, prevError: String): Unit = {
+ lastStatus: String): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
- auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error, prevStatus, prevError)
+ auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error, lastStatus)
auditEntityCounter += 1L
if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $auditEntityCounter")
@@ -406,7 +406,7 @@ object Settings {
writer.println(uuidsFailHeader)
val uuidList: List[AuditFailList] = auditUuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified, e.status))
uuidList.foreach { e =>
- writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error},${e.prevStatus},${e.prevError}")
+ writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error},${e.lastStatus}")
}
writer.flush()
writer.close()
[03/17] usergrid git commit: USERGRID-1044: make load entities
unlimited save invalid requests
Posted by mr...@apache.org.
USERGRID-1044: make load entities unlimited save invalid requests
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/1a6de02a
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/1a6de02a
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/1a6de02a
Branch: refs/heads/master
Commit: 1a6de02ab89f1e5272d1902003fb07085568c069
Parents: 7271935
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Oct 26 12:51:38 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Oct 26 12:51:38 2015 -0700
----------------------------------------------------------------------
stack/loadtests/runLoadEntitiesUnlimited.sh | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/1a6de02a/stack/loadtests/runLoadEntitiesUnlimited.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadEntitiesUnlimited.sh b/stack/loadtests/runLoadEntitiesUnlimited.sh
index 11ca0a4..c14b6b1 100755
--- a/stack/loadtests/runLoadEntitiesUnlimited.sh
+++ b/stack/loadtests/runLoadEntitiesUnlimited.sh
@@ -99,5 +99,6 @@ mvn gatling:execute \
-DuuidFilename=${UUID_FILENAME} \
-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
-DusergridRegion=${USERGRID_REGION} \
+-DsaveInvalidResponse=true \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
[15/17] usergrid git commit: USERGRID-1044: add audit delete test
(delete only, no verify)
Posted by mr...@apache.org.
USERGRID-1044: add audit delete test (delete only, no verify)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/247f7960
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/247f7960
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/247f7960
Branch: refs/heads/master
Commit: 247f79603d1775273a56274ce7810f1b8f1f5d29
Parents: 0fc1804
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Nov 2 07:34:42 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Nov 2 07:34:42 2015 -0800
----------------------------------------------------------------------
stack/loadtests/runAuditDeleteEntities.sh | 65 ++++++++++++++++++++
.../runAuditVerifyCollectionEntities.sh | 2 +-
.../apache/usergrid/enums/ScenarioType.scala | 3 +-
.../usergrid/scenarios/AuditScenarios.scala | 49 ++++++++++++++-
.../usergrid/simulations/AuditSimulation.scala | 4 +-
5 files changed, 118 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/runAuditDeleteEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditDeleteEntities.sh b/stack/loadtests/runAuditDeleteEntities.sh
new file mode 100755
index 0000000..ff92940
--- /dev/null
+++ b/stack/loadtests/runAuditDeleteEntities.sh
@@ -0,0 +1,65 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#RETRY_COUNT=
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME [USERGRID_REGION]]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+AUDIT_UUID_FILENAME="$3"
+FAILED_UUID_FILENAME="$4"
+[ "$#" -ge 5 ] && USERGRID_REGION="$5"
+
+shift $#
+
+SCENARIO_TYPE=auditDeleteEntities
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER} \
+-DadminPassword=${ADMIN_PASSWORD} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DretryCount=${RETRY_COUNT} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
+-DauditUuidFilename=${AUDIT_UUID_FILENAME} \
+-DfailedUuidFilename=${FAILED_UUID_FILENAME} \
+-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-DusergridRegion=${USERGRID_REGION} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/runAuditVerifyCollectionEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditVerifyCollectionEntities.sh b/stack/loadtests/runAuditVerifyCollectionEntities.sh
index 695c9ee..be50189 100755
--- a/stack/loadtests/runAuditVerifyCollectionEntities.sh
+++ b/stack/loadtests/runAuditVerifyCollectionEntities.sh
@@ -31,7 +31,7 @@ DELETE_AFTER_SUCCESSFUL_AUDIT=false
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME [DELETE_AFTER_SUCCESSFUL_AUDIT(true/false)[USERGRID_REGION]]]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME [DELETE_AFTER_SUCCESSFUL_AUDIT(true/false) [USERGRID_REGION]]]"
RAMP_USERS="$1"
RAMP_TIME="$2"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
index c86dc33..1cc1fa2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ScenarioType.scala
@@ -26,10 +26,11 @@ object ScenarioType {
val GetByNameSequential = "getByNameSequential"
val AuditGetCollectionEntities = "auditGetCollectionEntities"
val AuditVerifyCollectionEntities = "auditVerifyCollectionEntities"
+ val AuditDeleteEntities = "auditDeleteEntities"
val DoNothing = "doNothing"
val Values = Seq(GetAllByCursor,NameRandomInfinite,LoadEntities,DeleteEntities,UpdateEntities,UuidRandomInfinite,
- GetByNameSequential,AuditGetCollectionEntities,AuditVerifyCollectionEntities,DoNothing)
+ GetByNameSequential,AuditGetCollectionEntities,AuditVerifyCollectionEntities,AuditDeleteEntities,DoNothing)
def isValid(str: String): Boolean = {
Values.contains(str)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index cc59e0a..9ca479a 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -147,7 +147,6 @@ object AuditScenarios {
.get("/${collectionName}/${accessField}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
- .check()
.check(status.in(Seq(200,404)),status.saveAs(SessionVarStatus),extractAuditEntities(SessionVarCollectionEntities),
extractEntityUuid(SessionVarEntityUuid),extractEntityName(SessionVarEntityName)))
.exec(session => {
@@ -282,7 +281,7 @@ object AuditScenarios {
// }
}
- val verifyCollections = scenario("Verify collections")
+ val verifyAuditedEntities = scenario("Verify audited entities")
.exec(injectTokenIntoSession())
.exec(injectAuthType())
.asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
@@ -294,4 +293,50 @@ object AuditScenarios {
}
}
+ val deleteEntity = exec(
+ http("DELETE entity")
+ .delete("/${collectionName}/${accessField}")
+ .headers(Headers.authToken)
+ .headers(Headers.usergridRegionHeaders)
+ .check(status.saveAs(SessionVarStatus)))
+ .exec(session => {
+ val saveFailures = Settings.saveInvalidResponse
+ val status = session(SessionVarStatus).as[Int]
+ val collectionName = session(SessionVarCollectionName).as[String]
+ val modified = session("modified").as[String].toLong
+ val uuid = session("uuid").as[String]
+ val reqName = session("name").as[String]
+ val lastStatus = session("lastStatus").as[String]
+ if (status == 200) {
+ // success
+ Settings.incAuditSuccess()
+ session
+ } else if (status == 401 || status == 404) {
+ // didn't exist (currently returns 401, but 404 would be more appropriate)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFound", lastStatus)
+ Settings.incAuditEntryDeleteFailure()
+ session
+ } else if (saveFailures) {
+ // a different error
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"Error", lastStatus)
+ Settings.incAuditEntryDeleteFailure()
+ println(s"DELETE ERROR (status=$status): $collectionName.$reqName ($uuid)")
+ session.markAsFailed
+ } else {
+ session.markAsFailed
+ }
+
+ })
+
+ val deleteAuditedEntities = scenario("Delete audited entities")
+ .exec(injectTokenIntoSession())
+ .exec(injectAuthType())
+ .asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
+ feed(FeederGenerator.collectionCsvFeeder)
+ .doIf(session => session("validEntity").as[String] == "yes") {
+ tryMax(if (Settings.saveInvalidResponse) 1 else 1+Settings.retryCount) {
+ exec(deleteEntity)
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/247f7960/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
index bdec3dd..ac900fd 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
@@ -33,7 +33,8 @@ class AuditSimulation extends Simulation {
def getScenario(scenarioType: String): ScenarioBuilder = {
scenarioType match {
case ScenarioType.AuditGetCollectionEntities => AuditScenarios.getAllCollections
- case ScenarioType.AuditVerifyCollectionEntities => AuditScenarios.verifyCollections
+ case ScenarioType.AuditVerifyCollectionEntities => AuditScenarios.verifyAuditedEntities
+ case ScenarioType.AuditDeleteEntities => AuditScenarios.deleteAuditedEntities
}
}
@@ -66,6 +67,7 @@ class AuditSimulation extends Simulation {
val uuidDesc = Settings.scenarioType match {
case ScenarioType.AuditGetCollectionEntities => "found"
case ScenarioType.AuditVerifyCollectionEntities => "failed"
+ case ScenarioType.AuditDeleteEntities => "failed"
}
Settings.writeAuditUuidsToFile(uuidDesc)
}
[10/17] usergrid git commit: USERGRID-1044: print non-200s during
load (including body)
Posted by mr...@apache.org.
USERGRID-1044: print non-200s during load (including body)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/03897447
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/03897447
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/03897447
Branch: refs/heads/master
Commit: 0389744752f479d215f84ebbd5f25aa36d2108a4
Parents: 83d7a6f
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 14:59:10 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 14:59:10 2015 -0700
----------------------------------------------------------------------
.../usergrid/scenarios/EntityCollectionScenarios.scala | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/03897447/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index f37a2d2..d5b3101 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -42,6 +42,7 @@ object EntityCollectionScenarios {
val SessionVarStatus: String = "status"
val SessionVarUuid: String = "createUuid"
val SessionVarModified: String = "createModified"
+ val SessionVarBodyString: String = "bodyString"
def entityGetUrl(useCursor: Boolean): String = {
val url = s"/${Settings.collection}?" +
@@ -225,11 +226,12 @@ object EntityCollectionScenarios {
new ResponseWrapper(response) {
val contentType = response.header("content-type").getOrElse("").toLowerCase
val bodyStr = if (contentType.contains("json")) response.body.string else "[]"
+ if (bodyStr == "[]") { println(">>>>>>>>>>> USING EMPTY BODY") }
override val body = StringResponseBody(bodyStr, response.charset)
}
}
// 200 for success, 400 if already exists
- .check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
+ .check(status.saveAs(SessionVarStatus), bodyString.saveAs(SessionVarBodyString), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
val saveFailures = Settings.saveInvalidResponse
val status = session(SessionVarStatus).as[Int]
@@ -239,6 +241,10 @@ object EntityCollectionScenarios {
val entityName = session("entityName").as[String]
val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
val collectionName = session("collectionName").as[String]
+ if (status != 200) {
+ val bodyString = session(SessionVarBodyString).as[String]
+ println(s">>>>>>>> LOAD ERROR - Status: $status\nBody:\n$bodyString")
+ }
Settings.addUuid(uuid, collectionName, entityName, modified, status)
session
} else {
[11/17] usergrid git commit: USERGRID-1044: for org/app/collection
create, default to testConfig settings
Posted by mr...@apache.org.
USERGRID-1044: for org/app/collection create, default to testConfig settings
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fc23b79e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fc23b79e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fc23b79e
Branch: refs/heads/master
Commit: fc23b79e9a8a08bc57d25e3e60a9718304f8f1c0
Parents: 0389744
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 16:39:33 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 16:39:33 2015 -0700
----------------------------------------------------------------------
stack/loadtests/runOrgAppSetup.sh | 17 ++++++++++-------
1 file changed, 10 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fc23b79e/stack/loadtests/runOrgAppSetup.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runOrgAppSetup.sh b/stack/loadtests/runOrgAppSetup.sh
index 19936b7..b26eb37 100755
--- a/stack/loadtests/runOrgAppSetup.sh
+++ b/stack/loadtests/runOrgAppSetup.sh
@@ -29,6 +29,9 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
#ENTITY_TYPE=
#ENTITY_PREFIX=
#ENTITY_SEED= #may be overridden on command line
+#ORG= #may be overridden on command line
+#APP= #may be overridden on command line
+#COLLECTION= #may be overridden on command line
#RETRY_COUNT=
#ENTITY_PROGRESS_COUNT=
#CONSTANT_USERS_PER_SEC=
@@ -36,14 +39,14 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 ORG APP [COLLECTION [SANDBOX_COLLECTION (true/false)]]"
+[ "$#" -ge 1 ] || die "At least 1 argument required, $# provided. Example is $0 SANDBOX_COLLECTION(true/false) [ORG [APP [COLLECTION]]]"
-ORG="$1"
-APP="$2"
-COLLECTION="gatlingitems"
-[ "$#" -ge 3 ] && COLLECTION="$3"
-SANDBOX_COLLECTION=true
-[ "$#" -ge 4 ] && SANDBOX_COLLECTION="$4"
+
+SANDBOX_COLLECTION="$1"
+# org, app, and collection can come from testConfig.sh
+[ "$#" -ge 2 ] && ORG="$2"
+[ "$#" -ge 3 ] && APP="$3"
+[ "$#" -ge 4 ] && COLLECTION="$4"
shift $#
[08/17] usergrid git commit: USERGRID-1044: save previous status and
error in audit failure CSV
Posted by mr...@apache.org.
USERGRID-1044: save previous status and error in audit failure CSV
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/be0033ea
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/be0033ea
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/be0033ea
Branch: refs/heads/master
Commit: be0033eac3f715f63ec716a43ee4d28bffed33dc
Parents: 9158772
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 12:56:05 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 12:56:05 2015 -0700
----------------------------------------------------------------------
.../usergrid/scenarios/AuditScenarios.scala | 18 +++++++++++-------
.../org/apache/usergrid/settings/Settings.scala | 12 +++++++-----
2 files changed, 18 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be0033ea/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 7a93eb6..86c0b73 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -32,7 +32,7 @@ object AuditScenarios {
//The value for the cursor
val SessionVarCursor: String = "cursor"
- val SessionVarStatus: String = "status"
+ val SessionVarStatus: String = "newStatus"
val SessionVarEntityUuid: String = "entityUuid"
val SessionVarEntityName: String = "entityName"
val SessionVarDeletedUuid: String = "deletedUuid"
@@ -157,6 +157,8 @@ object AuditScenarios {
val modified = session("modified").as[String].toLong
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
+ val prevStatus = session("status").as[Int]
+ val prevError = session("error").as[String]
if (status == 200 || status == 404) {
val collectionEntities = session(SessionVarCollectionEntities).as[Seq[Any]]
val entityUuid = session(SessionVarEntityUuid).as[String]
@@ -164,12 +166,12 @@ object AuditScenarios {
val count = collectionEntities.length
if (count < 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll", prevStatus, prevError)
Settings.incAuditNotFoundAtAll()
println(s"NOT FOUND AT ALL: $collectionName.$reqName ($uuid)")
} else if (count > 1) {
// invalid
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -197,13 +199,13 @@ object AuditScenarios {
println(s"PAYLOAD NAME MISMATCH (DIRECT): requestedName=$reqName returnedName=$entityName")
}
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
Settings.incAuditNotFoundViaQuery()
println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
}
session
} else if (saveFailures) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", prevStatus, prevError)
session
} else {
session.markAsFailed
@@ -225,6 +227,8 @@ object AuditScenarios {
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
val modified = session("modified").as[String].toLong
+ val prevStatus = session("status").as[Int]
+ val prevError = session("error").as[String]
val collectionName = session(SessionVarCollectionName).as[String]
val entityUuid = session(SessionVarEntityUuid).as[String]
val entityName = session(SessionVarEntityName).as[String]
@@ -232,7 +236,7 @@ object AuditScenarios {
if (count < 1) {
// will check to see whether accessible directly
} else if (count > 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count", prevStatus, prevError)
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
@@ -264,7 +268,7 @@ object AuditScenarios {
// log even if technically successful -- we need to capture incorrect response
if (errorString != errorPrefix) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString, prevStatus, prevError)
}
Settings.incAuditSuccess()
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/be0033ea/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 1db0a55..e0a0ec9 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -291,9 +291,10 @@ object Settings {
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
val uuidsHeader = "collection,name,uuid,modified,status"
- val uuidsFailHeader = "collection,name,uuid,modified,status,error"
+ val uuidsFailHeader = "collection,name,uuid,modified,status,error,prevStatus,prevError"
case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int)
- case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int, var error: String)
+ case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long,
+ var status: Int, var error: String, var prevStatus: Int, var prevError: String)
//private var uuidMap: Map[Int, String] = Map()
private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
@@ -381,10 +382,11 @@ object Settings {
private var auditEntityCounter: Long = 0L
private var lastAuditEntityCountPrinted: Long = 0L
private var auditUuidList: mutable.MutableList[AuditFailList] = mutable.MutableList[AuditFailList]()
- def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String): Unit = {
+ def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String,
+ prevStatus: Int, prevError: String): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
- auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error)
+ auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error, prevStatus, prevError)
auditEntityCounter += 1L
if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $auditEntityCounter")
@@ -404,7 +406,7 @@ object Settings {
writer.println(uuidsFailHeader)
val uuidList: List[AuditFailList] = auditUuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified, e.status))
uuidList.foreach { e =>
- writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error}")
+ writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error},${e.prevStatus},${e.prevError}")
}
writer.flush()
writer.close()
[13/17] usergrid git commit: USERGRID-1044: print progress during
audit and log failures as such
Posted by mr...@apache.org.
USERGRID-1044: print progress during audit and log failures as such
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2a09bf15
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2a09bf15
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2a09bf15
Branch: refs/heads/master
Commit: 2a09bf15d8063199bf25b7e7f437929601f7193a
Parents: 2e70a72
Author: Mike Dunker <md...@apigee.com>
Authored: Wed Oct 28 07:28:51 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Wed Oct 28 07:28:51 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/datagenerators/FeederGenerator.scala | 8 ++++++++
.../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 6 +++---
.../usergrid/scenarios/EntityCollectionScenarios.scala | 8 +++++---
3 files changed, 16 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 38db19a..a9225b2 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -255,6 +255,9 @@ object FeederGenerator {
val csvLines = if (Settings.feedAuditUuids) Source.fromFile(Settings.feedAuditUuidFilename).getLines().toArray else Array[String]()
val csvLinesLen = csvLines.length
var counter = new AtomicInteger(0)
+ var entityCounter: Long = 0L
+ var lastEntityCountPrinted: Long = 0L
+ var entityProgressCount: Long = Settings.entityProgressCount
override def hasNext: Boolean = true
@@ -284,6 +287,11 @@ object FeederGenerator {
val accessField = if (uuid != "") uuid else name
val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
+ if (entityProgressCount > 0L && entityCounter >= lastEntityCountPrinted + entityProgressCount) {
+ println(s"CSV Entity: $entityCounter")
+ lastEntityCountPrinted = entityCounter
+ }
+
Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "lastStatus" -> lastStatus,
"validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField)
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 1ce2bc2..d1e2c4b 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -202,15 +202,15 @@ object AuditScenarios {
Settings.incAuditNotFoundViaQuery()
println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
}
- session
+ session.markAsFailed
} else if (saveFailures) {
Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure", lastStatus)
- session
+ session.markAsFailed
} else {
session.markAsFailed
}
- }).exitHereIfFailed
+ })
val getCollectionEntity = exec(
http("GET collection entity")
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2a09bf15/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index d5b3101..22b9f08 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -241,16 +241,18 @@ object EntityCollectionScenarios {
val entityName = session("entityName").as[String]
val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
if (status != 200) {
val bodyString = session(SessionVarBodyString).as[String]
println(s">>>>>>>> LOAD ERROR - Status: $status\nBody:\n$bodyString")
+ session.markAsFailed
+ } else {
+ session
}
- Settings.addUuid(uuid, collectionName, entityName, modified, status)
- session
} else {
session.markAsFailed
}
- }).exitHereIfFailed
+ })
}
)
[16/17] usergrid git commit: USERGRID-1044: update README
Posted by mr...@apache.org.
USERGRID-1044: update README
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/73dcaebd
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/73dcaebd
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/73dcaebd
Branch: refs/heads/master
Commit: 73dcaebd286fad3edffd0a8e523b31b042b061c2
Parents: 247f796
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Nov 2 08:54:18 2015 -0800
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Nov 2 08:54:18 2015 -0800
----------------------------------------------------------------------
stack/loadtests/README.md | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/73dcaebd/stack/loadtests/README.md
----------------------------------------------------------------------
diff --git a/stack/loadtests/README.md b/stack/loadtests/README.md
index ec34b42..9dc24e9 100644
--- a/stack/loadtests/README.md
+++ b/stack/loadtests/README.md
@@ -9,6 +9,9 @@ The test scripts are found in the top level loadtests directory. Look inside the
###testConfig.sh
Contains defaults that are used for all the other test scripts.
+###runAuditDeleteEntities.sh
+For a specified organization and given CSV file, delete all entities, writing those that fail to delete to another CSV file.
+
###runAuditGetAllAppCollectionEntities.sh
For a specified organization, finds all apps, and for each app, finds all collections and writes a CSV file line containing collection name, UUID, entity name, and modified timestamp for each entity in each collection.
@@ -16,7 +19,7 @@ For a specified organization, finds all apps, and for each app, finds all collec
For a specified organization and application, finds all collections and writes a CSV file line containing collection name, UUID, entity name, and modified timestamp for each entity in each collection.
###runAuditVerifyCollectionEntities.sh
-For a specified organization and given CSV file, verify that all entities are retrievable, writing those that fail to another CSV file.
+For a specified organization and given CSV file, verify that all entities are retrievable via query, writing those that fail to another CSV file, optionally deleting verified entities.
###runCollectionQueryTest.sh
For a given collection, retrieve all entities using a cursor and a query.
@@ -115,6 +118,13 @@ Defaults listed are those that are specified by the Usergrid Gatling code, not n
* getViaQuery (**false**) - retrieve entities via query instead of via name or uuid
* queryParams (**""**) - additional query parameters (currently used for get by entity or by name)
* csvFeedPattern (**"random"**) - pattern to use when feeding from a CSV ("random" is random, "circular" goes through CSV sequentially and restarts from beginning when it reaches the end)
+* unlimitedFeed (**false**) - continue loading with no limit on number of entities (forces interleavedWorkerFeed=true); hit CTRL-c to abort
+* flushCsv (**0**) - if > 0, flush CSV file when that number of CSV entries has been received
+* interleavedWorkerFeed (**false**) - for multiple gatling servers, shard the entities via interleaving rather than splitting into separate chunks; for example, for 10 workers over 1M entities, interleaving would cause one worker to have 1, 11, 21, 31, etc. instead of 1-100000
+* newCsvOnFlush (**false**) - when the output CSV file is flushed, create a new CSV file
+* deleteAfterSuccessfulAudit (**false**) - delete a record that is successfully verified via query
+* usergridRegion (**""**) - send specified region in the UsergridRegion header for all requests -- this allows Edge proxy to call different regions for testing
+* saveInvalidResponse (**false**) - on audit, do not retry but immediately save failure to output CSV file
The following settings are currently not used (were used by deprecated tests, but may be valid in the future):
[12/17] usergrid git commit: USERGRID-1044: add shutdown handler to
audit simulations
Posted by mr...@apache.org.
USERGRID-1044: add shutdown handler to audit simulations
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/2e70a72b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/2e70a72b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/2e70a72b
Branch: refs/heads/master
Commit: 2e70a72b99ac0a48e5f703a1b464fdad3c4d2665
Parents: fc23b79
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 16:45:02 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 16:45:02 2015 -0700
----------------------------------------------------------------------
.../apache/usergrid/simulations/AuditSimulation.scala | 11 +++++++++++
.../usergrid/simulations/ConfigurableSimulation.scala | 2 +-
2 files changed, 12 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e70a72b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
index 47323ea..bdec3dd 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
@@ -57,6 +57,10 @@ class AuditSimulation extends Simulation {
}
after {
+ endHandler
+ }
+
+ def endHandler: Unit = {
Settings.setTestEndTime()
if (Settings.captureAuditUuids) {
val uuidDesc = Settings.scenarioType match {
@@ -69,5 +73,12 @@ class AuditSimulation extends Simulation {
Settings.printAuditResults()
}
+ def abortedEarly: Unit = {
+ println(">>>>>>>>>>>>AUDIT ABORTED")
+ endHandler
+ }
+
+ sys addShutdownHook abortedEarly
+
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/2e70a72b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
index 9d4b05d..a3eb595 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
@@ -99,7 +99,7 @@ class ConfigurableSimulation extends Simulation {
Settings.printSettingsSummary(true)
}
- sys addShutdownHook(endHandler)
+ sys addShutdownHook endHandler
}
[07/17] usergrid git commit: USERGRID-1044: handle audit for CSV rows
that don't have UUIDs by accessing via name
Posted by mr...@apache.org.
USERGRID-1044: handle audit for CSV rows that don't have UUIDs by accessing via name
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9158772b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9158772b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9158772b
Branch: refs/heads/master
Commit: 9158772bc20befa8affa6c1bc10af032a07d0400
Parents: 4b2d003
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 12:46:48 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 12:46:48 2015 -0700
----------------------------------------------------------------------
.../org/apache/usergrid/datagenerators/FeederGenerator.scala | 5 ++++-
.../scala/org/apache/usergrid/scenarios/AuditScenarios.scala | 4 ++--
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9158772b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index b8a28d5..73b3d3c 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -280,8 +280,11 @@ object FeederGenerator {
val uuid = if (line != null) array(2) else ""
val modified = if (line != null) array(3) else ""
//println(s"$collectionName|$name|$uuid|$modified")
+ val accessField = if (uuid != "") uuid else name
+ val queryField = if (uuid != "") s"uuid='$uuid'" else s"name='$name'"
- Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "validEntity" -> validEntity)
+ Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified,
+ "validEntity" -> validEntity, "accessField" -> accessField, "queryField" -> queryField)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/9158772b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index f90348e..7a93eb6 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -144,7 +144,7 @@ object AuditScenarios {
val getCollectionEntityDirect = exec(
http("GET collection entity direct")
- .get("/${collectionName}/${uuid}")
+ .get("/${collectionName}/${accessField}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.check()
@@ -213,7 +213,7 @@ object AuditScenarios {
val getCollectionEntity = exec(
http("GET collection entity")
- .get("/${collectionName}?ql=uuid=${uuid}")
+ .get("/${collectionName}?ql=${queryField}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.check(status.is(200), status.saveAs(SessionVarStatus), jsonPath("$.count").optional.saveAs("count"),
[06/17] usergrid git commit: USERGRID-1044: attempt #2 to preprocess
response body
Posted by mr...@apache.org.
USERGRID-1044: attempt #2 to preprocess response body
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b2d0035
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b2d0035
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b2d0035
Branch: refs/heads/master
Commit: 4b2d00355e5645ab5c158b4c4f8e228037879af4
Parents: 2778f0a
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 27 11:29:16 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 27 11:29:16 2015 -0700
----------------------------------------------------------------------
.../usergrid/scenarios/EntityCollectionScenarios.scala | 13 ++++---------
1 file changed, 4 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b2d0035/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index bcdb49a..f37a2d2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -223,15 +223,10 @@ object EntityCollectionScenarios {
.transformResponse {
case response if response.isReceived =>
new ResponseWrapper(response) {
- override val body = {
- val contentType = response.header("content-type").getOrElse("").toLowerCase
- if (contentType.contains("json")) {
- StringResponseBody(response.body.string, response.charset)
- } else {
- StringResponseBody("{}", StandardCharsets.UTF_8)
- }
- }
- }
+ val contentType = response.header("content-type").getOrElse("").toLowerCase
+ val bodyStr = if (contentType.contains("json")) response.body.string else "[]"
+ override val body = StringResponseBody(bodyStr, response.charset)
+ }
}
// 200 for success, 400 if already exists
.check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
[02/17] usergrid git commit: USERGRID-1044: save status code and
error message for audit
Posted by mr...@apache.org.
USERGRID-1044: save status code and error message for audit
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7271935b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7271935b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7271935b
Branch: refs/heads/master
Commit: 7271935bb340a99f8961a99ca6f2d11b9e48b312
Parents: 1195ce8
Author: Mike Dunker <md...@apigee.com>
Authored: Mon Oct 26 09:56:28 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Mon Oct 26 09:56:28 2015 -0700
----------------------------------------------------------------------
.../runAuditGetAllAppCollectionEntities.sh | 6 +-
.../loadtests/runAuditGetCollectionEntities.sh | 6 +-
.../datagenerators/FeederGenerator.scala | 2 +-
.../usergrid/enums/ConfigProperties.scala | 6 +-
.../usergrid/scenarios/AuditScenarios.scala | 144 ++++++++++++-------
.../scenarios/EntityCollectionScenarios.scala | 74 ++++++----
.../org/apache/usergrid/settings/Settings.scala | 40 ++++--
7 files changed, 173 insertions(+), 105 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/runAuditGetAllAppCollectionEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditGetAllAppCollectionEntities.sh b/stack/loadtests/runAuditGetAllAppCollectionEntities.sh
index c8a30a3..f1304c0 100755
--- a/stack/loadtests/runAuditGetAllAppCollectionEntities.sh
+++ b/stack/loadtests/runAuditGetAllAppCollectionEntities.sh
@@ -31,10 +31,10 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [LATER_THAN_TIMESTAMP(ms)]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) UUID_FILENAME [LATER_THAN_TIMESTAMP(ms)]"
RAMP_USERS="$1"
RAMP_TIME="$2"
-AUDIT_UUID_FILENAME="$3"
+UUID_FILENAME="$3"
[ "$#" -ge 4 ] && LATER_THAN_TIMESTAMP="$4"
shift 3
@@ -61,7 +61,7 @@ mvn gatling:execute \
-DscenarioType=${SCENARIO_TYPE} \
-DrampUsers=${RAMP_USERS} \
-DrampTime=${RAMP_TIME} \
--DauditUuidFilename=${AUDIT_UUID_FILENAME} \
+-DuuidFilename=${UUID_FILENAME} \
-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/runAuditGetCollectionEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditGetCollectionEntities.sh b/stack/loadtests/runAuditGetCollectionEntities.sh
index 2ddc62b..7c87e25 100755
--- a/stack/loadtests/runAuditGetCollectionEntities.sh
+++ b/stack/loadtests/runAuditGetCollectionEntities.sh
@@ -32,10 +32,10 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [LATER_THAN_TIMESTAMP(ms)]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) UUID_FILENAME [LATER_THAN_TIMESTAMP(ms)]"
RAMP_USERS="$1"
RAMP_TIME="$2"
-AUDIT_UUID_FILENAME="$3"
+UUID_FILENAME="$3"
[ "$#" -ge 4 ] && LATER_THAN_TIMESTAMP="$4"
shift 3
@@ -61,7 +61,7 @@ mvn gatling:execute \
-DscenarioType=${SCENARIO_TYPE} \
-DrampUsers=${RAMP_USERS} \
-DrampTime=${RAMP_TIME} \
--DauditUuidFilename=${AUDIT_UUID_FILENAME} \
+-DuuidFilename=${UUID_FILENAME} \
-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index e08f8e8..b8a28d5 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -264,7 +264,7 @@ object FeederGenerator {
if (i >= csvLinesLen) return null
val line = csvLines(i)
- if (line != Settings.auditUuidsHeader) return line
+ if (line != Settings.uuidsHeader) return line
} while (true)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index cbb484c..7def02b 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -86,7 +86,8 @@ object ConfigProperties {
val InterleavedWorkerFeed = "interleavedWorkerFeed"
val NewCsvOnFlush = "newCsvOnFlush"
val DeleteAfterSuccessfulAudit = "deleteAfterSuccessfulAudit"
- val UsergridRegion = "usergridRegion";
+ val UsergridRegion = "usergridRegion"
+ val SaveInvalidResponse = "saveInvalidResponse"
val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
@@ -96,7 +97,7 @@ object ConfigProperties {
UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
EntityProgressCount,InjectionList,PrintFailedRequests,GetViaQuery,MultiPropertyPrefix,MultiPropertyCount,
MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern,UnlimitedFeed,FlushCsv,InterleavedWorkerFeed,
- NewCsvOnFlush,DeleteAfterSuccessfulAudit,UsergridRegion)
+ NewCsvOnFlush,DeleteAfterSuccessfulAudit,UsergridRegion,SaveInvalidResponse)
def isValid(str: String): Boolean = {
Values.contains(str)
@@ -171,6 +172,7 @@ object ConfigProperties {
case NewCsvOnFlush => false
case DeleteAfterSuccessfulAudit => false
case UsergridRegion => ""
+ case SaveInvalidResponse => false
}
} else {
null
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index 4477338..f90348e 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -32,6 +32,7 @@ object AuditScenarios {
//The value for the cursor
val SessionVarCursor: String = "cursor"
+ val SessionVarStatus: String = "status"
val SessionVarEntityUuid: String = "entityUuid"
val SessionVarEntityName: String = "entityName"
val SessionVarDeletedUuid: String = "deletedUuid"
@@ -59,15 +60,16 @@ object AuditScenarios {
.get(collectionGetUrl(false))
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
- .check(status.is(200),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
+ .check(status.is(200),status.saveAs(SessionVarStatus),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
.foreach("${" + SessionVarCollectionEntities + "}", "singleResult") {
exec(session => {
val resultObj = session("singleResult").as[Map[String,Any]]
+ val status = session(SessionVarStatus).as[Int]
val uuid = resultObj.getOrElse("uuid", "").asInstanceOf[String]
val entityName = resultObj.getOrElse("name", "").asInstanceOf[String]
- val modified = resultObj.getOrElse("modified", "-1").asInstanceOf[Long]
+ val modified = resultObj.getOrElse("modified", "0").asInstanceOf[Long]
val collectionName = session(SessionVarCollectionName).as[String]
- Settings.addAuditUuid(uuid, collectionName, entityName, modified)
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
session
})
}
@@ -77,15 +79,16 @@ object AuditScenarios {
.get(collectionGetUrl(true))
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
- .check(status.is(200),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
+ .check(status.is(200),status.saveAs(SessionVarStatus),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
.foreach("${" + SessionVarCollectionEntities + "}", "singleResult") {
exec(session => {
val resultObj = session("singleResult").as[Map[String,Any]]
+ val status = session(SessionVarStatus).as[Int]
val uuid = resultObj.getOrElse("uuid","").asInstanceOf[String]
val entityName = resultObj.getOrElse("name","").asInstanceOf[String]
val modified = resultObj.getOrElse("modified","-1").asInstanceOf[Long]
val collectionName = session(SessionVarCollectionName).as[String]
- Settings.addAuditUuid(uuid, collectionName, entityName, modified)
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
session
})
}
@@ -108,12 +111,12 @@ object AuditScenarios {
}
}
}
- }.exec { session =>
+ /*}.exec { session =>
// displays the content of the session in the console (debugging only)
println(session)
// return the original session
- session
+ session */
}
val deleteAuditedEntity = exec(
@@ -129,7 +132,11 @@ object AuditScenarios {
// successful
Settings.incAuditEntryDeleteSuccess()
} else {
+ val collectionName = session(SessionVarCollectionName).as[String]
+ val reqName = session("name").as[String]
+ val entityUuid = session(SessionVarEntityUuid).as[String]
Settings.incAuditEntryDeleteFailure()
+ println(s"DELETE FAILED: $collectionName:$reqName ($entityUuid)")
}
session
@@ -141,64 +148,80 @@ object AuditScenarios {
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
.check()
- .check(status.in(Seq(200,404)),extractAuditEntities(SessionVarCollectionEntities),
+ .check(status.in(Seq(200,404)),status.saveAs(SessionVarStatus),extractAuditEntities(SessionVarCollectionEntities),
extractEntityUuid(SessionVarEntityUuid),extractEntityName(SessionVarEntityName)))
.exec(session => {
+ val saveFailures = Settings.saveInvalidResponse
+ val status = session(SessionVarStatus).as[Int]
+ val collectionName = session(SessionVarCollectionName).as[String]
+ val modified = session("modified").as[String].toLong
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
- val modified = session("modified").as[String].toLong
- val collectionName = session(SessionVarCollectionName).as[String]
- val collectionEntities = session(SessionVarCollectionEntities).as[Seq[Any]]
- val entityUuid = session(SessionVarEntityUuid).as[String]
- val entityName = session(SessionVarEntityName).as[String]
+ if (status == 200 || status == 404) {
+ val collectionEntities = session(SessionVarCollectionEntities).as[Seq[Any]]
+ val entityUuid = session(SessionVarEntityUuid).as[String]
+ val entityName = session(SessionVarEntityName).as[String]
- val count = collectionEntities.length
- if (count < 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified)
- Settings.incAuditNotFoundAtAll()
- println(s"NOT FOUND AT ALL: $collectionName.$reqName ($uuid)")
- } else if (count > 1) {
- // invalid
- Settings.addAuditUuid(uuid, collectionName, reqName, modified)
- Settings.incAuditBadResponse()
- println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
- } else {
- // count == 1 -> found via direct access but not query
+ val count = collectionEntities.length
+ if (count < 1) {
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"NotFoundAtAll")
+ Settings.incAuditNotFoundAtAll()
+ println(s"NOT FOUND AT ALL: $collectionName.$reqName ($uuid)")
+ } else if (count > 1) {
+ // invalid
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count")
+ Settings.incAuditBadResponse()
+ println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
+ } else {
+ // count == 1 -> found via direct access but not query
- // will count as found directly even if there is a uuid or name mismatch
- if (entityUuid == null || entityUuid.isEmpty) {
- Settings.incAuditPayloadUuidError()
- println(s"PAYLOAD UUID MISSING (DIRECT): requestedUuid=$uuid")
- } else if (!uuid.equalsIgnoreCase(entityUuid)) {
- Settings.incAuditPayloadUuidError()
- println(s"PAYLOAD UUID MISMATCH (DIRECT): requestedUuid=$uuid returnedUuid=$entityUuid")
- }
- if (entityName == null || entityName.isEmpty) {
- Settings.incAuditPayloadNameError()
- println(s"PAYLOAD NAME MISSING (DIRECT): requestedName=$reqName")
- } else if (!reqName.equalsIgnoreCase(entityName)) {
- Settings.incAuditPayloadNameError()
- println(s"PAYLOAD NAME MISMATCH (DIRECT): requestedName=$reqName returnedName=$entityName")
- }
+ // will count as found directly even if there is a uuid or name mismatch
+ val errorPrefix = "DirectAccessSuccessful"
+ var errorString: String = errorPrefix
+ if (entityUuid == null || entityUuid.isEmpty) {
+ errorString += "|NoUuidReturned"
+ Settings.incAuditPayloadUuidError()
+ println(s"PAYLOAD UUID MISSING (DIRECT): requestedUuid=$uuid")
+ } else if (!uuid.equalsIgnoreCase(entityUuid)) {
+ errorString += "|ReturnedUuidMismatch"
+ Settings.incAuditPayloadUuidError()
+ println(s"PAYLOAD UUID MISMATCH (DIRECT): requestedUuid=$uuid returnedUuid=$entityUuid")
+ }
+ if (entityName == null || entityName.isEmpty) {
+ errorString += "|NoNameReturned"
+ Settings.incAuditPayloadNameError()
+ println(s"PAYLOAD NAME MISSING (DIRECT): requestedName=$reqName")
+ } else if (!reqName.equalsIgnoreCase(entityName)) {
+ errorString += "|ReturnedNameMismatch"
+ Settings.incAuditPayloadNameError()
+ println(s"PAYLOAD NAME MISMATCH (DIRECT): requestedName=$reqName returnedName=$entityName")
+ }
- Settings.addAuditUuid(uuid, collectionName, reqName, modified)
- Settings.incAuditNotFoundViaQuery()
- println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString)
+ Settings.incAuditNotFoundViaQuery()
+ println(s"NOT FOUND VIA QUERY: $collectionName.$reqName ($uuid)")
+ }
+ session
+ } else if (saveFailures) {
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, "Failure")
+ session
+ } else {
+ session.markAsFailed
}
- session
- })
+ }).exitHereIfFailed
val getCollectionEntity = exec(
http("GET collection entity")
.get("/${collectionName}?ql=uuid=${uuid}")
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
- .check(status.is(200),jsonPath("$.count").optional.saveAs("count"),
+ .check(status.is(200), status.saveAs(SessionVarStatus), jsonPath("$.count").optional.saveAs("count"),
extractAuditEntities(SessionVarCollectionEntities),
extractEntityUuid(SessionVarEntityUuid),extractEntityName(SessionVarEntityName)))
.exec(session => {
val count = session("count").as[String].toInt
+ val status = session(SessionVarStatus).as[Int]
val uuid = session("uuid").as[String]
val reqName = session("name").as[String]
val modified = session("modified").as[String].toLong
@@ -209,29 +232,40 @@ object AuditScenarios {
if (count < 1) {
// will check to see whether accessible directly
} else if (count > 1) {
- Settings.addAuditUuid(uuid, collectionName, reqName, modified)
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, s"QueryInvalidCount$count")
Settings.incAuditBadResponse()
println(s"INVALID RESPONSE (count=$count): $collectionName.$reqName ($uuid)")
} else {
// count == 1 -> success
// println(s"FOUND: $collectionName.$entityName ($uuid)")
- // will count as success even if there is a uuid or name mismatch
+ val errorPrefix = "QuerySuccessful"
+ var errorString: String = errorPrefix
if (entityUuid == null || entityUuid.isEmpty) {
+ errorString += "|NoUuidReturned"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISSING (QUERY): requestedUuid=$uuid")
} else if (!uuid.equalsIgnoreCase(entityUuid)) {
+ errorString += "|ReturnedUuidMismatch"
Settings.incAuditPayloadUuidError()
println(s"PAYLOAD UUID MISMATCH (QUERY): requestedUuid=$uuid returnedUuid=$entityUuid")
}
- if (entityName == null || entityName.isEmpty) {
- Settings.incAuditPayloadNameError()
- println(s"PAYLOAD NAME MISSING (QUERY): requestedName=$reqName")
- } else if (!reqName.equalsIgnoreCase(entityName)) {
- Settings.incAuditPayloadNameError()
- println(s"PAYLOAD NAME MISMATCH (QUERY): requestedName=$reqName returnedName=$entityName")
+ if (reqName != null && reqName != "") {
+ if (entityName == null || entityName.isEmpty) {
+ errorString += "|NoNameReturned"
+ Settings.incAuditPayloadNameError()
+ println(s"PAYLOAD NAME MISSING (QUERY): requestedName=$reqName")
+ } else if (!reqName.equalsIgnoreCase(entityName)) {
+ errorString += "|ReturnedNameMismatch"
+ Settings.incAuditPayloadNameError()
+ println(s"PAYLOAD NAME MISMATCH (QUERY): requestedName=$reqName returnedName=$entityName")
+ }
}
+ // log even if technically successful -- we need to capture incorrect response
+ if (errorString != errorPrefix) {
+ Settings.addAuditUuid(uuid, collectionName, reqName, modified, status, errorString)
+ }
Settings.incAuditSuccess()
}
@@ -252,7 +286,7 @@ object AuditScenarios {
.asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
feed(FeederGenerator.collectionCsvFeeder)
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(1+Settings.retryCount) {
+ tryMax(if (Settings.saveInvalidResponse) 1 else 1+Settings.retryCount) {
exec(getCollectionEntity)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index b56917a..ecd0c21 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -35,6 +35,7 @@ object EntityCollectionScenarios {
//The value for the cursor
val SessionVarCursor: String = "cursor"
+ val SessionVarStatus: String = "status"
val SessionVarUuid: String = "createUuid"
val SessionVarModified: String = "createModified"
@@ -216,15 +217,22 @@ object EntityCollectionScenarios {
.headers(Headers.usergridRegionHeaders)
.body(StringBody("""${entity}"""))
// 200 for success, 400 if already exists
- .check(status.in(Seq(200)), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
+ .check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
- val uuid = session(SessionVarUuid).as[String]
- val entityName = session("entityName").as[String]
- val modified = session(SessionVarModified).as[Long]
- val collectionName = session("collectionName").as[String]
- Settings.addUuid(uuid, collectionName, entityName, modified)
- session
- })
+ val saveFailures = Settings.saveInvalidResponse
+ val status = session(SessionVarStatus).as[Int]
+ Settings.addStatus(status)
+ if (saveFailures || status == 200) {
+ val uuid = if (status == 200) session(SessionVarUuid).as[String] else ""
+ val entityName = session("entityName").as[String]
+ val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
+ val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
+ session
+ } else {
+ session.markAsFailed
+ }
+ }).exitHereIfFailed
}
)
@@ -238,7 +246,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(1+Settings.retryCount) {
+ tryMax(if (Settings.saveInvalidResponse) 1 else 1+Settings.retryCount) {
exec(loadEntity)
}
}
@@ -309,15 +317,22 @@ object EntityCollectionScenarios {
.queryParamMap(Settings.queryParamMap)
.headers(Headers.authAnonymous)
.headers(Headers.usergridRegionHeaders)
- .check(status.is(200), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
+ .check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
- val uuid = session(SessionVarUuid).as[String]
- val entityName = session("entityName").as[String]
- val modified = session(SessionVarModified).as[Long]
- val collectionName = session("collectionName").as[String]
- Settings.addUuid(uuid, collectionName, entityName, modified)
- session
- })
+ val saveFailures = Settings.saveInvalidResponse
+ val status = session(SessionVarStatus).as[Int]
+ Settings.addStatus(status)
+ if (saveFailures || status == 200) {
+ val uuid = if (status == 200) session(SessionVarUuid).as[String] else ""
+ val entityName = session("entityName").as[String]
+ val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
+ val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
+ session
+ } else {
+ session.markAsFailed
+ }
+ }).exitHereIfFailed
}
)
@@ -328,15 +343,22 @@ object EntityCollectionScenarios {
.queryParamMap(Settings.queryParamMap)
.headers(Headers.authToken)
.headers(Headers.usergridRegionHeaders)
- .check(status.is(200), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
+ .check(status.saveAs(SessionVarStatus), extractEntityUuid(SessionVarUuid), extractEntityModified(SessionVarModified)))
.exec(session => {
- val uuid = session(SessionVarUuid).as[String]
- val entityName = session("entityName").as[String]
- val modified = session(SessionVarModified).as[Long]
- val collectionName = session("collectionName").as[String]
- Settings.addUuid(uuid, collectionName, entityName, modified)
- session
- })
+ val saveFailures = Settings.saveInvalidResponse
+ val status = session(SessionVarStatus).as[Int]
+ Settings.addStatus(status)
+ if (saveFailures || status == 200) {
+ val uuid = if (status == 200) session(SessionVarUuid).as[String] else ""
+ val entityName = session("entityName").as[String]
+ val modified = if (status == 200) session(SessionVarModified).as[Long] else 0
+ val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified, status)
+ session
+ } else {
+ session.markAsFailed
+ }
+ }).exitHereIfFailed
}
)
@@ -350,7 +372,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(1+Settings.retryCount) {
+ tryMax(if (Settings.saveInvalidResponse) 1 else 1+Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getEntityByNameSequentialAnonymous)
} {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7271935b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 2ecc3e3..1db0a55 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -188,6 +188,7 @@ object Settings {
val newCsvOnFlush:Boolean = initBoolSetting(ConfigProperties.NewCsvOnFlush)
val deleteAfterSuccessfulAudit:Boolean = initBoolSetting(ConfigProperties.DeleteAfterSuccessfulAudit)
val usergridRegion = initStrSetting(ConfigProperties.UsergridRegion)
+ val saveInvalidResponse = initBoolSetting(ConfigProperties.SaveInvalidResponse)
val multiPropertyPrefix = initStrSetting(ConfigProperties.MultiPropertyPrefix)
val multiPropertyCount:Int = initIntSetting(ConfigProperties.MultiPropertyCount)
@@ -289,22 +290,31 @@ object Settings {
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
- val auditUuidsHeader = "collection,name,uuid,modified"
- val uuidsHeader = "name,uuid"
- case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long)
+ val uuidsHeader = "collection,name,uuid,modified,status"
+ val uuidsFailHeader = "collection,name,uuid,modified,status,error"
+ case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int)
+ case class AuditFailList(var collection: String, var entityName: String, var uuid: String, var modified: Long, var status: Int, var error: String)
//private var uuidMap: Map[Int, String] = Map()
private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
+ private val statusCounts: mutable.Map[Int,Long] = mutable.Map[Int,Long]().withDefaultValue(0L)
private var entityCounter: Long = 0L
private var lastEntityCountPrinted: Long = 0L
private var flushCounter: Long = 0L
private var firstFlush: Boolean = true
private var numberFlushes: Long = 0L
private var uuidWriter: PrintWriter = null
- def addUuid(uuid: String, collection: String, entityName: String, modified: Long): Unit = {
+
+ def addStatus(status: Int): Unit = {
+ statusCounts.synchronized {
+ statusCounts(status) += 1L
+ }
+ }
+
+ def addUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int): Unit = {
if (captureUuids) {
uuidList.synchronized {
- uuidList += AuditList(collection, entityName, uuid, modified)
+ uuidList += AuditList(collection, entityName, uuid, modified, status)
entityCounter += 1L
flushCounter += 1L
if (logEntityProgress && (entityCounter >= lastEntityCountPrinted + entityProgressCount)) {
@@ -320,11 +330,11 @@ object Settings {
}
}
if (newCsvOnFlush || firstFlush) {
- uuidWriter.println(auditUuidsHeader)
+ uuidWriter.println(uuidsHeader)
}
val sortedUuidList: List[AuditList] = uuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified))
sortedUuidList.foreach { e =>
- uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+ uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status}")
}
uuidWriter.flush()
if (newCsvOnFlush) {
@@ -351,11 +361,11 @@ object Settings {
}
}
if (newCsvOnFlush || firstFlush) {
- uuidWriter.println(auditUuidsHeader)
+ uuidWriter.println(uuidsHeader)
}
val sortedUuidList: List[AuditList] = uuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified))
sortedUuidList.foreach { e =>
- uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+ uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status}")
}
uuidWriter.flush()
uuidWriter.close()
@@ -370,11 +380,11 @@ object Settings {
// key: uuid, value: collection
private var auditEntityCounter: Long = 0L
private var lastAuditEntityCountPrinted: Long = 0L
- private var auditUuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
- def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long): Unit = {
+ private var auditUuidList: mutable.MutableList[AuditFailList] = mutable.MutableList[AuditFailList]()
+ def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long, status: Int, error: String): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
- auditUuidList += AuditList(collection, entityName, uuid, modified)
+ auditUuidList += AuditFailList(collection, entityName, uuid, modified, status, error)
auditEntityCounter += 1L
if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $auditEntityCounter")
@@ -391,10 +401,10 @@ object Settings {
val fos = new FileOutputStream(captureAuditUuidFilename)
new PrintWriter(fos, false)
}
- writer.println(auditUuidsHeader)
- val uuidList: List[AuditList] = auditUuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified))
+ writer.println(uuidsFailHeader)
+ val uuidList: List[AuditFailList] = auditUuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified, e.status))
uuidList.foreach { e =>
- writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+ writer.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified},${e.status},${e.error}")
}
writer.flush()
writer.close()