You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/08/12 23:35:38 UTC
[1/6] incubator-usergrid git commit: USERGRID-871: simplify scripts
and add functionality to audit only records modified after a certain
timestamp
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-dev d7310c503 -> 1ff4774a3
USERGRID-871: simplify scripts and add functionality to audit only records modified after a certain timestamp
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/7253ce5f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/7253ce5f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/7253ce5f
Branch: refs/heads/two-dot-o-dev
Commit: 7253ce5fc60d0f985d64dbb970123826eb6f4621
Parents: c5e1623
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 4 16:43:05 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 4 16:43:05 2015 -0700
----------------------------------------------------------------------
.../loadtests/runAuditGetCollectionEntities.sh | 75 +++++------
.../runAuditVerifyCollectionEntities.sh | 68 +++++-----
stack/loadtests/runCollectionQueryTest.sh | 112 +++++++---------
stack/loadtests/runDeleteEntities.sh | 115 +++++++---------
.../loadtests/runGetEntitiesByNameSequential.sh | 112 ++++++++--------
stack/loadtests/runLoadEntities.sh | 127 ++++++++----------
stack/loadtests/runLoadSortableEntities.sh | 134 +++++++++----------
stack/loadtests/runRandomEntityByNameTest.sh | 113 +++++++---------
stack/loadtests/runRandomEntityByUuidTest.sh | 111 +++++++--------
stack/loadtests/runUpdateEntities.sh | 117 ++++++++--------
.../datagenerators/FeederGenerator.scala | 2 +-
.../usergrid/enums/ConfigProperties.scala | 11 +-
.../apache/usergrid/helpers/Extractors.scala | 2 +-
.../usergrid/scenarios/AuditScenarios.scala | 18 ++-
.../scenarios/EntityCollectionScenarios.scala | 64 +++++----
.../org/apache/usergrid/settings/Settings.scala | 101 +++++++++++---
.../usergrid/simulations/AuditSimulation.scala | 5 +-
.../simulations/ConfigurableSimulation.scala | 27 +++-
stack/loadtests/testConfig.sh | 57 ++++++++
19 files changed, 721 insertions(+), 650 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runAuditGetCollectionEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditGetCollectionEntities.sh b/stack/loadtests/runAuditGetCollectionEntities.sh
index b79b609..c47b88d 100755
--- a/stack/loadtests/runAuditGetCollectionEntities.sh
+++ b/stack/loadtests/runAuditGetCollectionEntities.sh
@@ -13,62 +13,55 @@
# limitations under the License.
#
-die() { echo "$@" 1>&2 ; exit 1; }
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 4 ] || die "At least 4 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME"
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#SEARCH_QUERY=
+#SEARCH_LIMIT=
+#LATER_THAN_TIMESTAMP= #may be overridden on command line
+#ENTITY_PROGRESS_COUNT=
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-AUDIT_UUID_FILENAME="$4"
+die() { echo "$@" 1>&2 ; exit 1; }
-shift 4
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [LATER_THAN_TIMESTAMP(ms)]"
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+AUDIT_UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && LATER_THAN_TIMESTAMP="$4"
-#Compile everything
-mvn compile
+shift 3
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-ORG=gatling
-APP=millionentities
SCENARIO_TYPE=auditGetCollectionEntities
-SEARCH_LIMIT=1000
-#SEARCH_QUERY=order%20by%20name
-SEARCH_QUERY=
-AUTH_TYPE=token
-TOKEN_TYPE=management
+#Compile everything
+mvn compile
#Execute the test
mvn gatling:execute \
--Dorg=${ORG} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
--DscenarioType=${SCENARIO_TYPE} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
-DauthType=${AUTH_TYPE} \
-DtokenType=${TOKEN_TYPE} \
--DauditUuidFilename=${AUDIT_UUID_FILENAME} \
--DsearchLimit=${SEARCH_LIMIT} \
-DsearchQuery=${SEARCH_QUERY} \
+-DsearchLimit=${SEARCH_LIMIT} \
+-DlaterThanTimestamp=${LATER_THAN_TIMESTAMP} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
+-DauditUuidFilename=${AUDIT_UUID_FILENAME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runAuditVerifyCollectionEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runAuditVerifyCollectionEntities.sh b/stack/loadtests/runAuditVerifyCollectionEntities.sh
index f0af5de..877e0ad 100755
--- a/stack/loadtests/runAuditVerifyCollectionEntities.sh
+++ b/stack/loadtests/runAuditVerifyCollectionEntities.sh
@@ -13,59 +13,51 @@
# limitations under the License.
#
-die() { echo "$@" 1>&2 ; exit 1; }
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#RETRY_COUNT=
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 4 ] || die "At least 4 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME]"
+die() { echo "$@" 1>&2 ; exit 1; }
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-AUDIT_UUID_FILENAME="$4"
-FAILED_UUID_FILENAME="$5"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) AUDIT_UUID_FILENAME [FAILED_UUID_FILENAME]"
-shift 5
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+AUDIT_UUID_FILENAME="$3"
+FAILED_UUID_FILENAME="$4"
+[ "$#" -ge 5 ] && LATER_THAN_TIMESTAMP="$5"
-#Compile everything
-mvn compile
+shift 4
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-ORG=gatling
-APP=millionentities
SCENARIO_TYPE=auditVerifyCollectionEntities
-AUTH_TYPE=token
-TOKEN_TYPE=management
+#Compile everything
+mvn compile
#Execute the test
mvn gatling:execute \
--Dorg=${ORG} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
--DscenarioType=${SCENARIO_TYPE} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
-DauthType=${AUTH_TYPE} \
-DtokenType=${TOKEN_TYPE} \
+-DretryCount=${RETRY_COUNT} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-DauditUuidFilename=${AUDIT_UUID_FILENAME} \
-DfailedUuidFilename=${FAILED_UUID_FILENAME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.AuditSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runCollectionQueryTest.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runCollectionQueryTest.sh b/stack/loadtests/runCollectionQueryTest.sh
index 85910e6..40b63f8 100755
--- a/stack/loadtests/runCollectionQueryTest.sh
+++ b/stack/loadtests/runCollectionQueryTest.sh
@@ -13,87 +13,75 @@
# limitations under the License.
#
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES=
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_SEED=
+#SEARCH_QUERY=
+#SEARCH_LIMIT= #may be overridden on command line
+#END_CONDITION_TYPE=
+#END_MINUTES=
+#END_REQUEST_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
die() { echo "$@" 1>&2 ; exit 1; }
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 6 ] || die "At least 6 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) SEARCH_LIMIT"
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [SEARCH_LIMIT]"
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-SEARCH_LIMIT="$6"
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && SEARCH_LIMIT="$3"
-shift 6
+shift $#
+
+SCENARIO_TYPE=getAllByCursor
#Compile everything
mvn compile
-#Set the app id to be a date epoch for uniqueness
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-#APP=$(date +%s)
-APP=millionentities
-COLLECTION=sortableentities
-SCENARIO_TYPE=getAllByCursor
-LOAD_ENTITIES=false
-NUM_ENTITIES=10000
-SKIP_SETUP=false
-SEARCH_QUERY=order%20by%20sortField%20desc
-ENTITY_TYPE=trivialSortable
-ENTITY_PREFIX=sortable
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
-
#Execute the test
mvn gatling:execute -Dorg=usergrid \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
--DentityPrefix=${ENTITY_PREFIX} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentitySeed=${ENTITY_SEED} \
-DsearchQuery=${SEARCH_QUERY} \
-DsearchLimit=${SEARCH_LIMIT} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
--DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
--DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
--DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
-DendConditionType=${END_CONDITION_TYPE} \
-DendMinutes=${END_MINUTES} \
-DendRequestCount=${END_REQUEST_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runDeleteEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runDeleteEntities.sh b/stack/loadtests/runDeleteEntities.sh
index 0883fa8..0d69bab 100755
--- a/stack/loadtests/runDeleteEntities.sh
+++ b/stack/loadtests/runDeleteEntities.sh
@@ -13,87 +13,76 @@
# limitations under the License.
#
-die() { echo "$@" 1>&2 ; exit 1; }
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 6 ] || die "At least 6 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES"
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_PREFIX=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-NUM_ENTITIES="$6"
+die() { echo "$@" 1>&2 ; exit 1; }
-shift 6
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
-#Compile everything
-mvn compile
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && NUM_ENTITIES="$3"
+[ "$#" -ge 4 ] && ENTITY_SEED="$4"
+[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+
+shift $#
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=trivialentities
SCENARIO_TYPE=deleteEntities
-# don't load entities as part of setup
-LOAD_ENTITIES=false
-SKIP_SETUP=true
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
-ENTITY_TYPE=trivial
-ENTITY_PREFIX=trivial
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
+#Compile everything
+mvn compile
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentityPrefix=${ENTITY_PREFIX} \
-DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
+-DretryCount=${RETRY_COUNT} \
-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
-DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
--DendConditionType=${END_CONDITION_TYPE} \
--DendMinutes=${END_MINUTES} \
--DendRequestCount=${END_REQUEST_COUNT} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runGetEntitiesByNameSequential.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runGetEntitiesByNameSequential.sh b/stack/loadtests/runGetEntitiesByNameSequential.sh
index d97edc2..f7b36b1 100755
--- a/stack/loadtests/runGetEntitiesByNameSequential.sh
+++ b/stack/loadtests/runGetEntitiesByNameSequential.sh
@@ -13,85 +13,79 @@
# limitations under the License.
#
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_PREFIX=
+#ENTITY_SEED=
+#RETRY_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
+
die() { echo "$@" 1>&2 ; exit 1; }
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 8 ] || die "At least 8 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES ENTITY_WORKER_NUM ENTITY_WORKER_COUNT [UUID_FILENAME]"
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [UUID_FILENAME [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-NUM_ENTITIES="$6"
-ENTITY_WORKER_NUM="$7"
-ENTITY_WORKER_COUNT="$8"
-UUID_FILENAME="$9"
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && NUM_ENTITIES="$3"
+[ "$#" -ge 4 ] && UUID_FILENAME="$4"
+[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
-shift 9
+shift $#
+
+SCENARIO_TYPE=getByNameSequential
#Compile everything
mvn compile
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=trivialentities
-SANDBOX_COLLECTION=false
-SCENARIO_TYPE=getByNameSequential
-LOAD_ENTITIES=false
-
-SKIP_SETUP=false
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
-ENTITY_TYPE=trivial
-ENTITY_PREFIX=trivial
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
-
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentityPrefix=${ENTITY_PREFIX} \
-DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
+-DretryCount=${RETRY_COUNT} \
-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
-DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
--DendConditionType=${END_CONDITION_TYPE} \
--DendMinutes=${END_MINUTES} \
--DendRequestCount=${END_REQUEST_COUNT} \
--DentityWorkerCount=${ENTITY_WORKER_COUNT} \
--DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-DuuidFilename=${UUID_FILENAME} \
--DsandboxCollection=${SANDBOX_COLLECTION} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runLoadEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadEntities.sh b/stack/loadtests/runLoadEntities.sh
index 1471d70..82045bb 100755
--- a/stack/loadtests/runLoadEntities.sh
+++ b/stack/loadtests/runLoadEntities.sh
@@ -13,95 +13,84 @@
# limitations under the License.
#
-die() { echo "$@" 1>&2 ; exit 1; }
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 8 ] || die "At least 8 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES ENTITY_WORKER_NUM ENTITY_WORKER_COUNT [UUID_FILENAME]"
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_TYPE=
+#ENTITY_PREFIX=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#ENTITY_PROGRESS_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-NUM_ENTITIES="$6"
-ENTITY_WORKER_NUM="$7"
-ENTITY_WORKER_COUNT="$8"
-UUID_FILENAME="$9"
+die() { echo "$@" 1>&2 ; exit 1; }
-shift 9
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]]"
-#Compile everything
-mvn compile
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && NUM_ENTITIES="$4"
+[ "$#" -ge 5 ] && ENTITY_SEED="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_NUM="$6"
+[ "$#" -ge 7 ] && ENTITY_WORKER_COUNT="$7"
+
+shift $#
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=trivialentities
-SANDBOX_COLLECTION=true
SCENARIO_TYPE=loadEntities
-# don't load entities as part of setup
+
+# don't load entities as part of setup (loading entities is the point of the test)
LOAD_ENTITIES=false
-SKIP_SETUP=false
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
-ENTITY_TYPE=trivial
-ENTITY_PREFIX=trivial
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
+#Compile everything
+mvn compile
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
--DloadEntities=${LOAD_ENTITIES} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentityType=${ENTITY_TYPE} \
-DentityPrefix=${ENTITY_PREFIX} \
-DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
+-DretryCount=${RETRY_COUNT} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
-DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
--DendConditionType=${END_CONDITION_TYPE} \
--DendMinutes=${END_MINUTES} \
--DendRequestCount=${END_REQUEST_COUNT} \
--DentityWorkerCount=${ENTITY_WORKER_COUNT} \
--DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-DuuidFilename=${UUID_FILENAME} \
--DsandboxCollection=${SANDBOX_COLLECTION} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runLoadSortableEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadSortableEntities.sh b/stack/loadtests/runLoadSortableEntities.sh
index 5289b6a..94f2a9a 100755
--- a/stack/loadtests/runLoadSortableEntities.sh
+++ b/stack/loadtests/runLoadSortableEntities.sh
@@ -11,97 +11,87 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#
-die() { echo "$@" 1>&2 ; exit 1; }
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 8 ] || die "At least 8 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES ENTITY_WORKER_NUM ENTITY_WORKER_COUNT [UUID_FILENAME]"
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-NUM_ENTITIES="$6"
-ENTITY_WORKER_NUM="$7"
-ENTITY_WORKER_COUNT="$8"
-UUID_FILENAME="$9"
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#ENTITY_PROGRESS_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
-shift 9
-#Compile everything
-mvn compile
+die() { echo "$@" 1>&2 ; exit 1; }
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=sortableentities
-SANDBOX_COLLECTION=true
-SCENARIO_TYPE=loadEntities
-# don't load entities as part of setup
-LOAD_ENTITIES=false
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]]"
-SKIP_SETUP=false
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && NUM_ENTITIES="$4"
+[ "$#" -ge 5 ] && ENTITY_SEED="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_NUM="$6"
+[ "$#" -ge 7 ] && ENTITY_WORKER_COUNT="$7"
+
+shift $#
+
+SCENARIO_TYPE=loadEntities
ENTITY_TYPE=trivialSortable
ENTITY_PREFIX=sortable
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
+
+# don't load entities as part of setup (loading entities is the point of the test)
+LOAD_ENTITIES=false
+
+#Compile everything
+mvn compile
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
--DloadEntities=${LOAD_ENTITIES} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
--DentityPrefix=${ENTITY_PREFIX} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
+-DretryCount=${RETRY_COUNT} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
-DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
--DendConditionType=${END_CONDITION_TYPE} \
--DendMinutes=${END_MINUTES} \
--DendRequestCount=${END_REQUEST_COUNT} \
--DentityWorkerCount=${ENTITY_WORKER_COUNT} \
--DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityType=${ENTITY_TYPE} \
+-DentityPrefix=${ENTITY_PREFIX} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-DuuidFilename=${UUID_FILENAME} \
--DsandboxCollection=${SANDBOX_COLLECTION} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runRandomEntityByNameTest.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runRandomEntityByNameTest.sh b/stack/loadtests/runRandomEntityByNameTest.sh
index 5967a1b..85a6457 100755
--- a/stack/loadtests/runRandomEntityByNameTest.sh
+++ b/stack/loadtests/runRandomEntityByNameTest.sh
@@ -13,85 +13,76 @@
# limitations under the License.
#
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_PREFIX=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#END_CONDITION_TYPE=
+#END_MINUTES=
+#END_REQUEST_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
die() { echo "$@" 1>&2 ; exit 1; }
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 5 ] || die "At least 5 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds)"
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED]]"
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && NUM_ENTITIES="$3"
+[ "$#" -ge 4 ] && ENTITY_SEED="$4"
-shift 5
+shift $#
+
+SCENARIO_TYPE=nameRandomInfinite
#Compile everything
mvn compile
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=trivialentities
-SCENARIO_TYPE=nameRandomInfinite
-LOAD_ENTITIES=false
-NUM_ENTITIES=10000
-SKIP_SETUP=false
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
-ENTITY_TYPE=trivial
-ENTITY_PREFIX=trivial
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
-
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentityPrefix=${ENTITY_PREFIX} \
-DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
--DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
--DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
--DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
+-DretryCount=${RETRY_COUNT} \
-DendConditionType=${END_CONDITION_TYPE} \
-DendMinutes=${END_MINUTES} \
-DendRequestCount=${END_REQUEST_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runRandomEntityByUuidTest.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runRandomEntityByUuidTest.sh b/stack/loadtests/runRandomEntityByUuidTest.sh
index 50f15e8..261b256 100755
--- a/stack/loadtests/runRandomEntityByUuidTest.sh
+++ b/stack/loadtests/runRandomEntityByUuidTest.sh
@@ -13,87 +13,72 @@
# limitations under the License.
#
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES=
+#SKIP_SETUP=
+#COLLECTION=
+#RETRY_COUNT=
+#END_CONDITION_TYPE=
+#END_MINUTES=
+#END_REQUEST_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
die() { echo "$@" 1>&2 ; exit 1; }
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 6 ] || die "At least 6 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) UUID_FILENAME"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) UUID_FILENAME"
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-UUID_FILENAME="$6"
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+UUID_FILENAME="$3"
-shift 6
+shift 3
+
+SCENARIO_TYPE=uuidRandomInfinite
#Compile everything
mvn compile
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=trivialentities
-SCENARIO_TYPE=uuidRandomInfinite
-LOAD_ENTITIES=false
-NUM_ENTITIES=10000
-SKIP_SETUP=false
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
-ENTITY_TYPE=trivial
-ENTITY_PREFIX=trivial
-ENTITY_SEED=1
-AUTH_TYPE=anonymous
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=10
-END_REQUEST_COUNT=100
-
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
--DentityPrefix=${ENTITY_PREFIX} \
--DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
--DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
--DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DskipSetup=${SKIP_SETUP} \
-Dcollection=${COLLECTION} \
--DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
+-DretryCount=${RETRY_COUNT} \
-DendConditionType=${END_CONDITION_TYPE} \
-DendMinutes=${END_MINUTES} \
-DendRequestCount=${END_REQUEST_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-DuuidFilename=${UUID_FILENAME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/runUpdateEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runUpdateEntities.sh b/stack/loadtests/runUpdateEntities.sh
index 85d529d..b8265d2 100755
--- a/stack/loadtests/runUpdateEntities.sh
+++ b/stack/loadtests/runUpdateEntities.sh
@@ -13,87 +13,78 @@
# limitations under the License.
#
-die() { echo "$@" 1>&2 ; exit 1; }
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
-####
-#This is a script to simplify running gatling tests. It will default several parameters, invoke the maven plugins
-#Then aggregate the results
-####
-[ "$#" -ge 6 ] || die "At least 6 arguments required, $# provided. Example is $0 URL RAMP_USERS RAMP_TIME(seconds) CONSTANT_USERS_PER_SEC, CONSTANT_USERS_DURATION(seconds) NUM_ENTITIES"
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_PREFIX=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
-URL="$1"
-RAMP_USERS="$2"
-RAMP_TIME="$3"
-CONSTANT_USERS_PER_SEC="$4"
-CONSTANT_USERS_DURATION="$5"
-NUM_ENTITIES="$6"
+die() { echo "$@" 1>&2 ; exit 1; }
-shift 6
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
-#Compile everything
-mvn compile
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && NUM_ENTITIES="$3"
+[ "$#" -ge 4 ] && ENTITY_SEED="$4"
+[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+
+shift $#
-#Set the app id to be a date epoch for uniqueness
-#APP=$(date +%s)
-ADMIN_USER=superuser
-ADMIN_PASSWORD=test
-CREATE_ORG=false
-ORG=gatling
-CREATE_APP=false
-APP=millionentities
-COLLECTION=trivialentities
SCENARIO_TYPE=updateEntities
-# don't load entities as part of setup
-LOAD_ENTITIES=false
+UPDATE_PROPERTY=updateProp
-SKIP_SETUP=true
-#SEARCH_QUERY=order%20by%20specials%20desc
-#SEARCH_LIMIT=1000
-ENTITY_TYPE=trivial
-ENTITY_PREFIX=trivial
-ENTITY_SEED=1
-AUTH_TYPE=token
-TOKEN_TYPE=management
-END_CONDITION_TYPE=minutesElapsed
-#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
-END_REQUEST_COUNT=100
+#Compile everything
+mvn compile
#Execute the test
mvn gatling:execute \
--DskipSetup=${SKIP_SETUP} \
--DcreateOrg=${CREATE_ORG} \
--Dorg=${ORG} \
--DcreateApp=${CREATE_APP} \
--Dapp=${APP} \
--Dbaseurl=${URL} \
+-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
-DnumEntities=${NUM_ENTITIES} \
--DentityType=${ENTITY_TYPE} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
-DentityPrefix=${ENTITY_PREFIX} \
-DentitySeed=${ENTITY_SEED} \
--DrampUsers=${RAMP_USERS} \
--DrampTime=${RAMP_TIME} \
+-DretryCount=${RETRY_COUNT} \
-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
--Dcollection=${COLLECTION} \
-DscenarioType=${SCENARIO_TYPE} \
--DauthType=${AUTH_TYPE} \
--DtokenType=${TOKEN_TYPE} \
--DendConditionType=${END_CONDITION_TYPE} \
--DendMinutes=${END_MINUTES} \
--DendRequestCount=${END_REQUEST_COUNT} \
+-DupdateProperty=${UPDATE_PROPERTY} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
-
-
-#Now move all the reports
-#AGGREGATE_DIR="target/aggregate-$(date +%s)"
-
-#mkdir -p ${AGGREGATE_DIR}
-
-#copy to the format of target/aggregate(date)/(simnulationame)-simulation.log
-#find target -name "simulation.log" -exec cp {} ${AGGREGATE_DIR}/$(basename $(dirname {} ))-simulation.log \;
-
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 61641de..6b7c76f 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -127,7 +127,7 @@ object FeederGenerator {
var nameArray: ArrayBuffer[Map[String, String]] = new ArrayBuffer[Map[String, String]]
- for (entityCount <- 1 to numEntities) {
+ for (_ <- 1 to numEntities) {
nameArray += Map("entityName" -> prefix.concat(Utils.generateRandomInt(0, 100000000).toString))
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index 293cc9d..03581ba 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -26,7 +26,7 @@ object ConfigProperties {
val App = "app"
val AdminUser = "adminUser"
val AdminPassword = "adminPassword"
- val BaseUrl = "baseurl"
+ val BaseUrl = "baseUrl"
val AuthType = "authType"
val TokenType = "tokenType"
val SkipSetup = "skipSetup"
@@ -71,13 +71,17 @@ object ConfigProperties {
val FailedUuidFilename = "failedUuidFilename"
val SandboxCollection = "sandboxCollection"
val PurgeUsers = "purgeUsers"
+ val RetryCount = "retryCount"
+ val LaterThanTimestamp = "laterThanTimestamp"
+ val EntityProgressCount = "entityProgressCount"
val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
NumDevices,Collection,RampTime,Throttle,RpsTarget,RpsRampTime,HoldDuration,PushNotifier,PushProvider,EntityPrefix,
EntityType,EntitySeed,SearchLimit,SearchQuery,EndConditionType,EndMinutes,EndRequestCount,OrgCreationUsername,
OrgCreationName,OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum,
- UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers)
+ UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
+ EntityProgressCount)
def isValid(str: String): Boolean = {
Values.contains(str)
@@ -136,6 +140,9 @@ object ConfigProperties {
case FailedUuidFilename => "/tmp/dummy.csv"
case SandboxCollection => false
case PurgeUsers => 100
+ case RetryCount => 5
+ case LaterThanTimestamp => 0L
+ case EntityProgressCount => 10000L
}
} else {
null
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index 221c0a7..8e43368 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -71,7 +71,7 @@ object Extractors {
* @param saveAsName The name to use when saving to the session
*/
def maybeExtractEntities(saveAsName: String) = {
- jsonPath("$..entities").ofType[Seq[Any]].transformOption(extract => {
+ jsonPath("$.entities").ofType[Seq[Any]].transformOption(extract => {
extract.orElse(Some(Seq()))
}).saveAs(saveAsName)
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index eaf14a9..c951dcf 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -37,9 +37,14 @@ object AuditScenarios {
val SessionVarCollectionUuids: String = "collectionUuids"
def collectionGetUrl(useCursor: Boolean): String = {
+ val searchQuery =
+ // later than timestamp replaces query
+ if (Settings.laterThanTimestamp > 0) s"modified%20gte%20${Settings.laterThanTimestamp}"
+ else if (Settings.searchQuery != "") s"${Settings.searchQuery}"
+ else ""
val url = "/${" + SessionVarCollectionName + "}?" +
(if (useCursor) "cursor=${" + SessionVarCursor + "}&" else "") +
- (if (Settings.searchQuery != "") s"ql=${Settings.searchQuery}&" else "") +
+ (if (searchQuery != "") s"ql=$searchQuery&" else "") +
(if (Settings.searchLimit > 0) s"limit=${Settings.searchLimit}&" else "")
// remove trailing & or ?
@@ -51,7 +56,7 @@ object AuditScenarios {
http("GET collections")
.get(collectionGetUrl(false))
.headers(Headers.authToken)
- .check(status.is(200), extractCollectionUuids(SessionVarCollectionUuids), maybeExtractCursor(SessionVarCursor)))
+ .check(status.is(200), extractCollectionUuids(SessionVarCollectionUuids),maybeExtractCursor(SessionVarCursor)))
.foreach("${" + SessionVarCollectionUuids + "}", "singleResult") {
exec(session => {
val resultObj = session("singleResult").as[Map[String,Any]]
@@ -68,7 +73,7 @@ object AuditScenarios {
http("GET collections")
.get(collectionGetUrl(true))
.headers(Headers.authToken)
- .check(status.is(200), extractCollectionUuids(SessionVarCollectionUuids), maybeExtractCursor(SessionVarCursor)))
+ .check(status.is(200), extractCollectionUuids(SessionVarCollectionUuids),maybeExtractCursor(SessionVarCursor)))
.foreach("${" + SessionVarCollectionUuids + "}", "singleResult") {
exec(session => {
val resultObj = session("singleResult").as[Map[String,Any]]
@@ -91,9 +96,12 @@ object AuditScenarios {
session
}
.doIf(session => session("validEntity").as[String] == "yes") {
- exec(getCollectionsWithoutCursor)
- .asLongAs(stringParamExists(SessionVarCursor)) {
+ tryMax(Settings.retryCount) {
+ exec(getCollectionsWithoutCursor)
+ }.asLongAs(stringParamExists(SessionVarCursor)) {
+ tryMax(Settings.retryCount) {
exec(getCollectionsWithCursor)
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index a760931..eb7fce2 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -121,18 +121,22 @@ object EntityCollectionScenarios {
.exec(injectAuthType())
.doIfOrElse(_ => Settings.endConditionType == EndConditionType.MinutesElapsed) {
asLongAs(_ => Settings.continueMinutesTest) {
- doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
- exec(getRandomEntityAnonymous)
- } {
- exec(getRandomEntityWithToken)
+ tryMax(Settings.retryCount) {
+ doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
+ exec(getRandomEntityAnonymous)
+ } {
+ exec(getRandomEntityWithToken)
+ }
}
}
} {
repeat(_ => Settings.endRequestCount.toInt) {
- doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
- exec(getRandomEntityAnonymous)
- } {
- exec(getRandomEntityWithToken)
+ tryMax(Settings.retryCount) {
+ doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
+ exec(getRandomEntityAnonymous)
+ } {
+ exec(getRandomEntityWithToken)
+ }
}
}
}
@@ -159,30 +163,34 @@ object EntityCollectionScenarios {
.exec(injectAuthType())
.doIfOrElse(_ => Settings.endConditionType == EndConditionType.MinutesElapsed) {
asLongAs(_ => Settings.continueMinutesTest) {
- feed(uuidFeeder())
- /*.exec{
+ feed(uuidFeeder())
+ /*.exec{
session => println(s"UUID: ${session("uuid").as[String]}")
session
- }*/
- .doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
- exec(getRandomEntityByUuidAnonymous)
- } {
- exec(getRandomEntityByUuidWithToken)
+ }*/
+ .tryMax(Settings.retryCount) {
+ doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
+ exec(getRandomEntityByUuidAnonymous)
+ } {
+ exec(getRandomEntityByUuidWithToken)
+ }
}
}
} {
repeat(_ => Settings.endRequestCount.toInt) {
- feed(uuidFeeder())
- /*.exec {
+ feed(uuidFeeder())
+ /*.exec {
session => println(s"UUID: ${session("uuid").as[String]}")
session
- }*/
- .doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
- exec(getRandomEntityByUuidAnonymous)
- } {
- exec(getRandomEntityByUuidWithToken)
- }
- }
+ }*/
+ .tryMax(Settings.retryCount) {
+ doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
+ exec(getRandomEntityByUuidAnonymous)
+ } {
+ exec(getRandomEntityByUuidWithToken)
+ }
+ }
+ }
}
/*
@@ -213,7 +221,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(5) {
+ tryMax(Settings.retryCount) {
exec(loadEntity)
}
}
@@ -240,7 +248,9 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- exec(deleteEntity)
+ tryMax(Settings.retryCount) {
+ exec(deleteEntity)
+ }
}
}
@@ -309,7 +319,7 @@ object EntityCollectionScenarios {
session
}*/
.doIf(session => session("validEntity").as[String] == "yes") {
- tryMax(5) {
+ tryMax(Settings.retryCount) {
doIfOrElse(_ => Settings.authType == AuthType.Anonymous) {
exec(getEntityByNameSequentialAnonymous)
} {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 61231d5..08e1ae2 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -17,6 +17,7 @@
package org.apache.usergrid.settings
import java.io.{PrintWriter, FileOutputStream}
+import java.util.concurrent.TimeUnit
import java.util.concurrent.atomic.AtomicInteger
import javax.xml.bind.DatatypeConverter
import io.gatling.http.Predef._
@@ -58,6 +59,15 @@ object Settings {
ConfigProperties.getDefault(cfg).asInstanceOf[Int]
}
+ def initLongSetting(cfg: String): Long = {
+ val longSetting:java.lang.Long = java.lang.Long.getLong(cfg)
+
+ if (longSetting != null)
+ longSetting.toLong
+ else
+ ConfigProperties.getDefault(cfg).asInstanceOf[Long]
+ }
+
// load configuration settings via property or default
val org = initStrSetting(ConfigProperties.Org)
val app = initStrSetting(ConfigProperties.App)
@@ -132,6 +142,11 @@ object Settings {
val orgCreationName = if (cfgOrgCreationName == "") orgCreationUsername else cfgOrgCreationName
val orgCreationPassword = initStrSetting(ConfigProperties.OrgCreationPassword)
+ val retryCount:Int = initIntSetting(ConfigProperties.RetryCount)
+ val laterThanTimestamp:Long = initLongSetting(ConfigProperties.LaterThanTimestamp)
+ val entityProgressCount:Long = initLongSetting(ConfigProperties.EntityProgressCount)
+ private val logEntityProgress: Boolean = entityProgressCount > 0L
+
// Entity update
val updateProperty = initStrSetting(ConfigProperties.UpdateProperty)
val updateValue = initStrSetting(ConfigProperties.UpdateValue)
@@ -213,12 +228,7 @@ object Settings {
val captureAuditUuids = (scenarioType == ScenarioType.AuditGetCollectionEntities) ||
(scenarioType == ScenarioType.AuditVerifyCollectionEntities && captureAuditUuidFilename != dummyAuditFailedCsv)
- //val feedUuids = uuidFilename != dummyTestCsv && scenarioType == ScenarioType.UuidRandomInfinite
- //val feedAuditUuids = uuidFilename != dummyAuditCsv && scenarioType == ScenarioType.AuditVerifyCollectionEntities
- // val captureUuids = uuidFilename != dummyTestCsv && (scenarioType == ScenarioType.LoadEntities || scenarioType == ScenarioType.GetByNameSequential)
-
- //val captureAuditUuids = (auditUuidFilename != dummyAuditCsv && scenarioType == ScenarioType.AuditGetCollectionEntities) ||
- // (failedUuidFilename != dummyAuditFailedCsv && scenarioType == ScenarioType.AuditVerifyCollectionEntities)
+ /*
println(s"feedUuids=$feedUuids")
println(s"feedUuidFilename=$feedUuidFilename")
println(s"feedAuditUuids=$feedAuditUuids")
@@ -227,14 +237,22 @@ object Settings {
println(s"captureUuidFilename=$captureUuidFilename")
println(s"captureAuditUuids=$captureAuditUuids")
println(s"captureAuditUuidFilename=$captureAuditUuidFilename")
+ */
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
private var uuidMap: Map[Int, String] = Map()
+ private var entityCounter: Long = 0
+ private var lastEntityCountPrinted: Long = 0
def addUuid(num: Int, uuid: String): Unit = {
if (captureUuids) {
uuidMap.synchronized {
uuidMap += (num -> uuid)
+ entityCounter += 1
+ if (logEntityProgress && (entityCounter >= lastEntityCountPrinted + entityProgressCount)) {
+ println(s"Entity: $entityCounter")
+ lastEntityCountPrinted = entityCounter
+ }
}
}
// println(s"UUID: ${name},${uuid}")
@@ -262,11 +280,18 @@ object Settings {
case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long)
// key: uuid, value: collection
+ private var auditEntityCounter: Long = 0
+ private var lastAuditEntityCountPrinted: Long = 0
private var auditUuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
auditUuidList += AuditList(collection, entityName, uuid, modified)
+ auditEntityCounter += 1
+ if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
+ println(s"Entity: $auditEntityCounter")
+ lastAuditEntityCountPrinted = auditEntityCounter
+ }
}
}
}
@@ -297,6 +322,7 @@ object Settings {
}
private var testStartTime: Long = System.currentTimeMillis()
+ private var testEndTime: Long = 0
def getTestStartTime: Long = {
testStartTime
@@ -306,6 +332,10 @@ object Settings {
testStartTime = System.currentTimeMillis()
}
+ def setTestEndTime(): Unit = {
+ testEndTime = System.currentTimeMillis()
+ }
+
def continueMinutesTest: Boolean = {
(System.currentTimeMillis() - testStartTime) < (endMinutes.toLong*60L*1000L)
}
@@ -333,46 +363,87 @@ object Settings {
val countBadResponse = countAuditBadResponse.get
val countTotal = countSuccess + countNotFound + countBadResponse
+ val seconds = ((testEndTime - testStartTime) / 1000).toInt
+ val s:Int = seconds % 60
+ val m:Int = (seconds/60) % 60
+ val h:Int = seconds/(60*60)
+ val elapsedStr = f"$h%d:$m%02d:$s%02d"
+
println()
println("-----------------------------------------------------------------------------")
println("AUDIT RESULTS")
println("-----------------------------------------------------------------------------")
println()
- println(s"Successful: ${countSuccess}")
- println(s"Not Found: ${countNotFound}")
- println(s"Bad Response: ${countBadResponse}")
- println(s"Total: ${countTotal}")
+ println(s"Successful: $countSuccess")
+ println(s"Not Found: $countNotFound")
+ println(s"Bad Response: $countBadResponse")
+ println(s"Total: $countTotal")
+ println()
+ println(s"Start Timestamp(ms): $testStartTime")
+ println(s"End Timestamp(ms): $testEndTime")
+ println(s"Elapsed Time: $elapsedStr")
println()
println("-----------------------------------------------------------------------------")
println()
}
}
- def printSettingsSummary(): Unit = {
+ def printSettingsSummary(afterTest: Boolean): Unit = {
val authTypeStr = authType + (if (authType == AuthType.Token) s"($tokenType)" else "")
val endConditionStr = if (endConditionType == EndConditionType.MinutesElapsed) s"$endMinutes minutes elapsed" else s"$endRequestCount requests"
+ val seconds = ((testEndTime - testStartTime) / 1000).toInt
+ val s:Int = seconds % 60
+ val m:Int = (seconds/60) % 60
+ val h:Int = seconds/(60*60)
+ val elapsedStr = f"$h%d:$m%02d:$s%02d"
println()
println("-----------------------------------------------------------------------------")
println("SIMULATION SETTINGS")
println("-----------------------------------------------------------------------------")
println()
+ println(s"ScenarioType:$scenarioType AuthType:$authTypeStr")
+ println()
+ println(s"BaseURL:$baseUrl")
println(s"Org:$org App:$app Collection:$collection")
println(s"CreateOrg:$createOrg CreateApp:$createApp LoadEntities:$loadEntities")
- println(s"ScenarioType:$scenarioType AuthType:$authTypeStr")
+ println(s"SandboxCollection:$sandboxCollection SkipSetup:$skipSetup")
+ println(s"AuthType:$authType TokenType:$tokenType AdminUser:$adminUser")
println()
- println(s"Entity Type:$entityType Prefix:$entityPrefix")
+ println(s"EntityType:$entityType Prefix:$entityPrefix RetryCount:$retryCount")
+ if (scenarioType == ScenarioType.AuditGetCollectionEntities && laterThanTimestamp > 0) {
+ if (laterThanTimestamp > 0) println(s"SearchLimit:$searchLimit OnlyForEntriesAtOrLater:$laterThanTimestamp")
+ } else {
+ println(s"SearchLimit:$searchLimit SearchQuery:$searchQuery")
+ }
println()
println(s"Overall: NumEntities:$totalNumEntities Seed:$overallEntitySeed Workers:$entityWorkerCount")
println(s"Worker: NumEntities:$numEntities Seed:$entitySeed WorkerNum:$entityWorkerNum")
println()
println(s"Ramp: Users:$rampUsers Time:$rampTime")
println(s"Constant: UsersPerSec:$constantUsersPerSec Time:$constantUsersDuration")
- println(s"End Condition:$endConditionStr")
+ println(s"EndCondition:$endConditionStr")
+ println()
+ if (feedUuids) println(s"Feed CSV: $feedUuidFilename")
+ if (feedAuditUuids) println(s"Audit Feed CSV: $feedAuditUuidFilename")
+ if (captureUuids) println(s"Capture CSV:$captureUuidFilename")
+ if (captureAuditUuids) {
+ if (scenarioType == ScenarioType.AuditVerifyCollectionEntities)
+ println(s"Audit Capture CSV (failed entities):$captureAuditUuidFilename")
+ else
+ println(s"Audit Capture CSV:$captureAuditUuidFilename")
+ }
+ println()
+ println()
+ if (afterTest) {
+ println(s"TestStarted:$testStartTime TestEnded:$testEndTime Elapsed: $elapsedStr")
+ } else {
+ println(s"TestStarted:$testStartTime")
+ }
println()
println("-----------------------------------------------------------------------------")
println()
}
- printSettingsSummary()
+ printSettingsSummary(false)
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
index 48b5f60..caeb9ec 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/AuditSimulation.scala
@@ -37,9 +37,9 @@ class AuditSimulation extends Simulation {
}
before{
+ Settings.setTestStartTime()
}
- Settings.setTestStartTime()
if (ScenarioType.isValid(Settings.scenarioType)) {
val scenario: ScenarioBuilder = getScenario(Settings.scenarioType)
setUp(
@@ -53,6 +53,7 @@ class AuditSimulation extends Simulation {
}
after {
+ Settings.setTestEndTime()
if (Settings.captureAuditUuids) {
val uuidDesc = Settings.scenarioType match {
case ScenarioType.AuditGetCollectionEntities => "found"
@@ -60,7 +61,7 @@ class AuditSimulation extends Simulation {
}
Settings.writeAuditUuidsToFile(uuidDesc)
}
- Settings.printSettingsSummary()
+ Settings.printSettingsSummary(true)
Settings.printAuditResults()
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
index e52e3d8..ac4cfd4 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
@@ -17,12 +17,15 @@
package org.apache.usergrid.simulations
import io.gatling.core.Predef._
+import io.gatling.core.controller.inject.InjectionStep
import io.gatling.core.structure.ScenarioBuilder
import org.apache.usergrid.enums.ScenarioType
import org.apache.usergrid.helpers.Setup
import org.apache.usergrid.scenarios.EntityCollectionScenarios
import org.apache.usergrid.settings.Settings
+import scala.collection.mutable
+
/**
* Configurable simulations.
*
@@ -58,25 +61,37 @@ class ConfigurableSimulation extends Simulation {
println("Skipping setup")
}
if (Settings.sandboxCollection) Setup.sandboxCollection()
+ Settings.setTestStartTime()
}
- Settings.setTestStartTime()
if (ScenarioType.isValid(Settings.scenarioType)) {
val scenario: ScenarioBuilder = getScenario(Settings.scenarioType)
+ var stepCount:Int = 0
+ if (Settings.rampUsers > 0) stepCount += 1
+ if (Settings.constantUsersPerSec > 0) stepCount += 1
+ val injectStepList = new mutable.ArraySeq[InjectionStep](stepCount)
+ var currentStep = 0
+ if (Settings.rampUsers > 0) {
+ injectStepList(currentStep) = rampUsers(Settings.rampUsers) over Settings.rampTime
+ currentStep += 1
+ }
+ if (Settings.constantUsersPerSec > 0) {
+ injectStepList(currentStep) = constantUsersPerSec(Settings.constantUsersPerSec) during Settings.constantUsersDuration
+ currentStep += 1
+ }
setUp(
scenario
- .inject(
- rampUsers(Settings.rampUsers) over Settings.rampTime,
- constantUsersPerSec(Settings.constantUsersPerSec) during Settings.constantUsersDuration
- ).protocols(Settings.httpConf.acceptHeader("application/json"))
+ .inject(injectStepList)
+ .protocols(Settings.httpConf.acceptHeader("application/json"))
)
} else {
println(s"scenarioType ${Settings.scenarioType} not found.")
}
after {
+ Settings.setTestEndTime()
if (Settings.captureUuids) Settings.writeUuidsToFile()
- Settings.printSettingsSummary()
+ Settings.printSettingsSummary(true)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/7253ce5f/stack/loadtests/testConfig.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/testConfig.sh b/stack/loadtests/testConfig.sh
new file mode 100755
index 0000000..1e20637
--- /dev/null
+++ b/stack/loadtests/testConfig.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+####
+# Configuration for tests
+####
+
+URL="http://localhost:8080"
+ADMIN_USER=superuser
+ADMIN_PASSWORD=test
+
+ENTITY_WORKER_NUM=1
+ENTITY_WORKER_COUNT=1
+
+ORG=gatling
+APP=millionentities
+
+AUTH_TYPE=token
+TOKEN_TYPE=management
+
+CREATE_ORG=false
+CREATE_APP=false
+LOAD_ENTITIES=false
+SANDBOX_COLLECTION=true
+NUM_ENTITIES=10000
+SKIP_SETUP=false
+
+COLLECTION=thousand
+ENTITY_TYPE=trivialSortable
+ENTITY_PREFIX=sortable
+ENTITY_SEED=1
+
+SEARCH_QUERY=order%20by%20sortField%20desc
+SEARCH_LIMIT=1000
+RETRY_COUNT=5
+LATER_THAN_TIMESTAMP=0
+ENTITY_PROGRESS_COUNT=1000
+
+END_CONDITION_TYPE=minutesElapsed
+#END_CONDITION_TYPE=requestCount
+END_MINUTES=2
+END_REQUEST_COUNT=100
+
+CONSTANT_USERS_PER_SEC=0
+CONSTANT_USERS_DURATION=10
[3/6] incubator-usergrid git commit: USERGRID-871: fix AuditVerify
Posted by gr...@apache.org.
USERGRID-871: fix AuditVerify
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/211fe1aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/211fe1aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/211fe1aa
Branch: refs/heads/two-dot-o-dev
Commit: 211fe1aaa00d4d7951240e6b587ed1be6fb1c0bf
Parents: dc9dafe
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 4 23:58:00 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 4 23:58:00 2015 -0700
----------------------------------------------------------------------
.../datagenerators/FeederGenerator.scala | 7 ++-
.../apache/usergrid/helpers/Extractors.scala | 11 +++-
.../usergrid/scenarios/AuditScenarios.scala | 57 ++++++++++----------
3 files changed, 43 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/211fe1aa/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 6b7c76f..55c0bfe 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -272,9 +272,12 @@ object FeederGenerator {
val validEntity = if (line == null) "no" else "yes"
val array = if (line != null) line.split(",") else null
val collectionName = if (line != null) array(0) else ""
- val uuid = if (line != null) array(1) else ""
+ val name = if (line != null) array(1) else ""
+ val uuid = if (line != null) array(2) else ""
+ val modified = if (line != null) array(3) else ""
+ //println(s"$collectionName|$name|$uuid|$modified")
- Map("collectionName" -> collectionName, "uuid" -> uuid, "validEntity" -> validEntity)
+ Map("collectionName" -> collectionName, "name" -> name, "uuid" -> uuid, "modified" -> modified, "validEntity" -> validEntity)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/211fe1aa/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index 8e43368..d93c430 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -51,13 +51,20 @@ object Extractors {
}
/**
- * Will extract the uuids from the get collection response.
+ * Will extract the audit entities from the get collection response.
*/
- def extractCollectionUuids(saveAsName: String) = {
+ def extractAuditEntities(saveAsName: String) = {
jsonPath("$.entities[*]").ofType[Map[String,Any]].findAll.transformOption(extract => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
}
/**
+ * Will extract the audit entities from the get collection response.
+ */
+ def extractAuditEntity(saveAsName: String) = {
+ jsonPath("$.entities[0]").ofType[Map[String,Any]].findAll.transformOption(extract => { extract.orElse(Some(Seq.empty)) }).saveAs(saveAsName)
+ }
+
+ /**
* tries to extract the cursor from the session, if it exists, it returns true. if it's the default, returns false
* @param nameInSession The name of the variable in the session
* @return
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/211fe1aa/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index c951dcf..cdaa89b 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -34,7 +34,7 @@ object AuditScenarios {
val SessionVarCursor: String = "cursor"
val SessionVarUuid: String = "entityUuid"
val SessionVarCollectionName: String = "collectionName"
- val SessionVarCollectionUuids: String = "collectionUuids"
+ val SessionVarCollectionEntities: String = "collectionEntities"
def collectionGetUrl(useCursor: Boolean): String = {
val searchQuery =
@@ -56,8 +56,8 @@ object AuditScenarios {
http("GET collections")
.get(collectionGetUrl(false))
.headers(Headers.authToken)
- .check(status.is(200), extractCollectionUuids(SessionVarCollectionUuids),maybeExtractCursor(SessionVarCursor)))
- .foreach("${" + SessionVarCollectionUuids + "}", "singleResult") {
+ .check(status.is(200),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
+ .foreach("${" + SessionVarCollectionEntities + "}", "singleResult") {
exec(session => {
val resultObj = session("singleResult").as[Map[String,Any]]
val uuid = resultObj("uuid").asInstanceOf[String]
@@ -73,8 +73,8 @@ object AuditScenarios {
http("GET collections")
.get(collectionGetUrl(true))
.headers(Headers.authToken)
- .check(status.is(200), extractCollectionUuids(SessionVarCollectionUuids),maybeExtractCursor(SessionVarCursor)))
- .foreach("${" + SessionVarCollectionUuids + "}", "singleResult") {
+ .check(status.is(200),extractAuditEntities(SessionVarCollectionEntities),maybeExtractCursor(SessionVarCursor)))
+ .foreach("${" + SessionVarCollectionEntities + "}", "singleResult") {
exec(session => {
val resultObj = session("singleResult").as[Map[String,Any]]
val uuid = resultObj("uuid").asInstanceOf[String]
@@ -110,30 +110,29 @@ object AuditScenarios {
http("GET collection entity")
.get("/${collectionName}?ql=uuid=${uuid}")
.headers(Headers.authToken)
- .check(status.is(200), jsonPath("$.count").saveAs("count")))
- .exec(session => {
- val count = session("count").as[String].toInt
- val uuid = session("uuid").as[String]
- val entityName = session("name").as[String]
- val modified = session("modified").as[Long]
- val collectionName = session(SessionVarCollectionName).as[String]
+ .check(status.is(200),jsonPath("$.count").optional.saveAs("count"),extractAuditEntities(SessionVarCollectionEntities)))
+ .exec(session => {
+ val count = session("count").as[String].toInt
+ val uuid = session("uuid").as[String]
+ val entityName = session("name").as[String]
+ val modified = session("modified").as[String].toLong
+ val collectionName = session(SessionVarCollectionName).as[String]
- // save items not found
- if (count < 1) {
- Settings.addAuditUuid(uuid, collectionName, entityName, modified)
- Settings.incAuditNotFound()
- println(s"NOT FOUND: $collectionName.$entityName $uuid")
- } else if (count > 1) {
- Settings.addAuditUuid(uuid, collectionName, entityName, modified)
- Settings.incAuditBadResponse()
- println(s"INVALID RESPONSE (count=$count): $collectionName.$entityName $uuid")
- } else {
- // println(s"FOUND: $collectionName.$entityName $uuid")
- Settings.incAuditSuccess()
- }
+ if (count < 1) {
+ Settings.addAuditUuid(uuid, collectionName, entityName, modified)
+ Settings.incAuditNotFound()
+ println(s"NOT FOUND: $collectionName.$entityName $uuid")
+ } else if (count > 1) {
+ Settings.addAuditUuid(uuid, collectionName, entityName, modified)
+ Settings.incAuditBadResponse()
+ println(s"INVALID RESPONSE (count=$count): $collectionName.$entityName $uuid")
+ } else {
+ // println(s"FOUND: $collectionName.$entityName $uuid")
+ Settings.incAuditSuccess()
+ }
- session
- })
+ session
+ })
val verifyCollections = scenario("Verify collections")
.exec(injectTokenIntoSession())
@@ -141,7 +140,9 @@ object AuditScenarios {
.asLongAs(session => session("validEntity").asOption[String].map(validEntity => validEntity != "no").getOrElse[Boolean](true)) {
feed(FeederGenerator.collectionCsvFeeder)
.doIf(session => session("validEntity").as[String] == "yes") {
- exec(getCollectionEntity)
+ tryMax(Settings.retryCount) {
+ exec(getCollectionEntity)
+ }
}
}
[6/6] incubator-usergrid git commit: Merge branch 'two-dot-o-dev' of
https://github.com/mdunker/incubator-usergrid into two-dot-o-dev
Posted by gr...@apache.org.
Merge branch 'two-dot-o-dev' of https://github.com/mdunker/incubator-usergrid into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1ff4774a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1ff4774a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1ff4774a
Branch: refs/heads/two-dot-o-dev
Commit: 1ff4774a37694ed892daf0f67aac649e7a61d700
Parents: d7310c5 93fb85a
Author: GERey <gr...@apigee.com>
Authored: Wed Aug 12 14:23:35 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Wed Aug 12 14:23:35 2015 -0700
----------------------------------------------------------------------
.../loadtests/runAuditGetCollectionEntities.sh | 75 +++++------
.../runAuditVerifyCollectionEntities.sh | 68 +++++-----
stack/loadtests/runCollectionQueryTest.sh | 112 +++++++---------
.../runCustomInjectRandomEntityByUuidTest.sh | 107 +++++++++++++++
stack/loadtests/runDeleteEntities.sh | 115 +++++++---------
.../loadtests/runGetEntitiesByNameSequential.sh | 112 ++++++++--------
stack/loadtests/runLoadEntities.sh | 127 ++++++++----------
stack/loadtests/runLoadNoNameEntities.sh | 97 ++++++++++++++
stack/loadtests/runLoadSortableEntities.sh | 134 +++++++++----------
stack/loadtests/runRandomEntityByNameTest.sh | 113 +++++++---------
stack/loadtests/runRandomEntityByUuidTest.sh | 111 +++++++--------
stack/loadtests/runUpdateEntities.sh | 117 ++++++++--------
.../datagenerators/FeederGenerator.scala | 18 ++-
.../usergrid/enums/ConfigProperties.scala | 13 +-
.../usergrid/enums/EndConditionType.scala | 3 +-
.../apache/usergrid/helpers/Extractors.scala | 13 +-
.../usergrid/scenarios/AuditScenarios.scala | 71 +++++-----
.../scenarios/EntityCollectionScenarios.scala | 64 +++++----
.../org/apache/usergrid/settings/Settings.scala | 103 +++++++++++---
.../usergrid/simulations/AuditSimulation.scala | 5 +-
.../simulations/ConfigurableSimulation.scala | 33 ++++-
.../simulations/CustomInjectionSimulation.scala | 107 +++++++++++++++
stack/loadtests/testConfig.sh | 59 ++++++++
23 files changed, 1091 insertions(+), 686 deletions(-)
----------------------------------------------------------------------
[4/6] incubator-usergrid git commit: USERGRID-871: new custom
injection test plus loading no-name entities
Posted by gr...@apache.org.
USERGRID-871: new custom injection test plus loading no-name entities
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d3174e6d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d3174e6d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d3174e6d
Branch: refs/heads/two-dot-o-dev
Commit: d3174e6d7474e6d257cbe66ef996bf76b3ea86af
Parents: 211fe1a
Author: Mike Dunker <mi...@calbears.net>
Authored: Tue Aug 11 17:03:24 2015 -0700
Committer: Mike Dunker <mi...@calbears.net>
Committed: Tue Aug 11 17:03:24 2015 -0700
----------------------------------------------------------------------
.../runCustomInjectRandomEntityByUuidTest.sh | 107 +++++++++++++++++++
stack/loadtests/runLoadNoNameEntities.sh | 97 +++++++++++++++++
.../datagenerators/FeederGenerator.scala | 9 +-
.../usergrid/enums/ConfigProperties.scala | 4 +-
.../usergrid/enums/EndConditionType.scala | 3 +-
.../usergrid/scenarios/AuditScenarios.scala | 6 +-
.../org/apache/usergrid/settings/Settings.scala | 2 +-
.../simulations/ConfigurableSimulation.scala | 6 ++
.../simulations/CustomInjectionSimulation.scala | 107 +++++++++++++++++++
stack/loadtests/testConfig.sh | 6 +-
10 files changed, 335 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/runCustomInjectRandomEntityByUuidTest.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runCustomInjectRandomEntityByUuidTest.sh b/stack/loadtests/runCustomInjectRandomEntityByUuidTest.sh
new file mode 100755
index 0000000..3164a9d
--- /dev/null
+++ b/stack/loadtests/runCustomInjectRandomEntityByUuidTest.sh
@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#
+# Injection list:
+# type(arg1,arg2,...);type(arg1,...)
+#
+# types:
+# rampUsers(int numUsers, int overSeconds)
+# constantUsersPerSec(double numUsersPerSec, int duringSeconds)
+# constantUsersPerSecRandomized(double numUsersPerSec, int duringSeconds)
+# atOnceUsers(int numUsers)
+# rampUsersPerSec(double numUsersPerSec, int totalUsers, int duringSeconds)
+# rampUsersPerSecRandomized(double numUsersPerSec, int totalUsers, int duringSeconds)
+# heavisideUsers(int numUsers, int overSeconds)
+# nothingFor(int seconds)
+#
+# Example: rampUsers(30,120);nothingFor(120);atOnceUsers(20)
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#LOAD_ENTITIES=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES=
+#SKIP_SETUP=
+#COLLECTION=
+#RETRY_COUNT=
+#END_CONDITION_TYPE=
+#END_MINUTES=
+#END_REQUEST_COUNT=
+#INJECTION_LIST=
+
+helpMsg() {
+ echo "At least 2 arguments required, $# provided. Example is $0 INJECTION_LIST UUID_FILENAME" 1>&2
+ echo "Injection types:" 1>&2
+ echo " rampUsers(int numUsers, int overSeconds)" 1>&2
+ echo " constantUsersPerSec(double numUsersPerSec, int duringSeconds)" 1>&2
+ echo " constantUsersPerSecRandomized(double numUsersPerSec, int duringSeconds)" 1>&2
+ echo " atOnceUsers(int numUsers)" 1>&2
+ echo " rampUsersPerSec(double numUsersPerSec, int totalUsers, int duringSeconds)" 1>&2
+ echo " rampUsersPerSecRandomized(double numUsersPerSec, int totalUsers, int duringSeconds)" 1>&2
+ echo " heavisideUsers(int numUsers, int overSeconds)" 1>&2
+ echo " nothingFor(int seconds)" 1>&2
+ exit 1
+}
+
+[ "$#" -ge 2 ] || helpMsg
+
+INJECTION_LIST="$1"
+UUID_FILENAME="$2"
+
+shift 2
+
+SCENARIO_TYPE=uuidRandomInfinite
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER} \
+-DadminPassword=${ADMIN_PASSWORD} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
+-DnumEntities=${NUM_ENTITIES} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
+-DretryCount=${RETRY_COUNT} \
+-DendConditionType=${END_CONDITION_TYPE} \
+-DendMinutes=${END_MINUTES} \
+-DendRequestCount=${END_REQUEST_COUNT} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DuuidFilename=${UUID_FILENAME} \
+-DinjectionList=${INJECTION_LIST} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.CustomInjectionSimulation
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/runLoadNoNameEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadNoNameEntities.sh b/stack/loadtests/runLoadNoNameEntities.sh
new file mode 100755
index 0000000..4acf1cc
--- /dev/null
+++ b/stack/loadtests/runLoadNoNameEntities.sh
@@ -0,0 +1,97 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #may be overridden on command line
+#SKIP_SETUP=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#ENTITY_PROGRESS_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && NUM_ENTITIES="$4"
+[ "$#" -ge 5 ] && ENTITY_SEED="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_NUM="$6"
+[ "$#" -ge 7 ] && ENTITY_WORKER_COUNT="$7"
+
+shift $#
+
+SCENARIO_TYPE=loadEntities
+ENTITY_TYPE=trivialSortable
+ENTITY_PREFIX=""
+COLLECTION=nonames
+
+# don't load entities as part of setup (loading entities is the point of the test)
+LOAD_ENTITIES=false
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER} \
+-DadminPassword=${ADMIN_PASSWORD} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
+-DnumEntities=${NUM_ENTITIES} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
+-DentitySeed=${ENTITY_SEED} \
+-DretryCount=${RETRY_COUNT} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DentityType=${ENTITY_TYPE} \
+-DentityPrefix=${ENTITY_PREFIX} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
+-DuuidFilename=${UUID_FILENAME} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 55c0bfe..9546e2d 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -211,7 +211,7 @@ object FeederGenerator {
* This feeder will serve data forever, but validEntity will be set to "no" when data has run out. Each user can
* then exit in a controlled fashion.
*/
- def generateCustomEntityFeeder(numEntities: Int, entityType: String, prefix: String, seed: Int = 1): Feeder[String] =
+ def generateCustomEntityFeeder(numEntities: Int, entityType: String, prefix: String = "", seed: Int = 1): Feeder[String] =
new Feeder[String] {
var counter = new AtomicInteger(0)
@@ -221,8 +221,9 @@ object FeederGenerator {
override def next(): Map[String, String] = {
val i = counter.getAndIncrement()
val seededVal = i + seed
- val entityName = prefix.concat(seededVal.toString)
- val entity = EntityDataGenerator.generateEntity(entityType, entityName)
+ val noPrefix = prefix == null || prefix == ""
+ val entityName = if (noPrefix) seededVal.toString else prefix.concat(seededVal.toString)
+ val entity = EntityDataGenerator.generateEntity(entityType, if (noPrefix) null else entityName)
val entityUrl = Settings.baseCollectionUrl + "/" + entityName
val validEntity = if (i >= numEntities) "no" else "yes"
@@ -282,7 +283,7 @@ object FeederGenerator {
}
def generateCustomEntityInfiniteFeeder(seed: Int = Settings.entitySeed, entityType: String = Settings.entityType, prefix: String = Settings.entityPrefix): Iterator[String] = {
- Iterator.from(seed).map(i=>EntityDataGenerator.generateEntity(entityType, prefix.concat(i.toString)))
+ Iterator.from(seed).map(i=>EntityDataGenerator.generateEntity(entityType, if (prefix == null || prefix == "") null else prefix.concat(i.toString)))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index 03581ba..230f58b 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -74,6 +74,7 @@ object ConfigProperties {
val RetryCount = "retryCount"
val LaterThanTimestamp = "laterThanTimestamp"
val EntityProgressCount = "entityProgressCount"
+ val InjectionList = "injectionList"
val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
@@ -81,7 +82,7 @@ object ConfigProperties {
EntityType,EntitySeed,SearchLimit,SearchQuery,EndConditionType,EndMinutes,EndRequestCount,OrgCreationUsername,
OrgCreationName,OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum,
UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
- EntityProgressCount)
+ EntityProgressCount,InjectionList)
def isValid(str: String): Boolean = {
Values.contains(str)
@@ -143,6 +144,7 @@ object ConfigProperties {
case RetryCount => 5
case LaterThanTimestamp => 0L
case EntityProgressCount => 10000L
+ case InjectionList => "rampUsers(10,60)"
}
} else {
null
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/enums/EndConditionType.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/EndConditionType.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/EndConditionType.scala
index 6d77dd1..5dd78ad 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/EndConditionType.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/EndConditionType.scala
@@ -19,8 +19,9 @@ package org.apache.usergrid.enums
object EndConditionType {
val MinutesElapsed = "minutesElapsed"
val RequestCount = "requestCount"
+ val Unlimited = "unlimited"
- val Values = Seq(MinutesElapsed,RequestCount)
+ val Values = Seq(MinutesElapsed,RequestCount,Unlimited)
def isValid(str: String): Boolean = {
Values.contains(str)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
index cdaa89b..b3d19b4 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/AuditScenarios.scala
@@ -121,13 +121,13 @@ object AuditScenarios {
if (count < 1) {
Settings.addAuditUuid(uuid, collectionName, entityName, modified)
Settings.incAuditNotFound()
- println(s"NOT FOUND: $collectionName.$entityName $uuid")
+ println(s"NOT FOUND: $collectionName.$entityName ($uuid)")
} else if (count > 1) {
Settings.addAuditUuid(uuid, collectionName, entityName, modified)
Settings.incAuditBadResponse()
- println(s"INVALID RESPONSE (count=$count): $collectionName.$entityName $uuid")
+ println(s"INVALID RESPONSE (count=$count): $collectionName.$entityName ($uuid)")
} else {
- // println(s"FOUND: $collectionName.$entityName $uuid")
+ // println(s"FOUND: $collectionName.$entityName ($uuid)")
Settings.incAuditSuccess()
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 08e1ae2..6c226d5 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -128,7 +128,6 @@ object Settings {
val overallEntitySeed = initIntSetting(ConfigProperties.EntitySeed)
val searchLimit:Int = initIntSetting(ConfigProperties.SearchLimit)
val searchQuery = initStrSetting(ConfigProperties.SearchQuery)
- println(s"searchQuery=${searchQuery}")
val endConditionType = initStrSetting(ConfigProperties.EndConditionType)
val endMinutes:Int = initIntSetting(ConfigProperties.EndMinutes)
val endRequestCount:Int = initIntSetting(ConfigProperties.EndRequestCount)
@@ -146,6 +145,7 @@ object Settings {
val laterThanTimestamp:Long = initLongSetting(ConfigProperties.LaterThanTimestamp)
val entityProgressCount:Long = initLongSetting(ConfigProperties.EntityProgressCount)
private val logEntityProgress: Boolean = entityProgressCount > 0L
+ val injectionList = initStrSetting(ConfigProperties.InjectionList)
// Entity update
val updateProperty = initStrSetting(ConfigProperties.UpdateProperty)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
index ac4cfd4..66aaac0 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/ConfigurableSimulation.scala
@@ -89,10 +89,16 @@ class ConfigurableSimulation extends Simulation {
}
after {
+ endHandler
+ }
+
+ def endHandler: Unit = {
Settings.setTestEndTime()
if (Settings.captureUuids) Settings.writeUuidsToFile()
Settings.printSettingsSummary(true)
}
+ sys addShutdownHook(endHandler)
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/CustomInjectionSimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/CustomInjectionSimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/CustomInjectionSimulation.scala
new file mode 100755
index 0000000..9d27173
--- /dev/null
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/CustomInjectionSimulation.scala
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.simulations
+
+import io.gatling.core.Predef._
+import io.gatling.core.controller.inject.InjectionStep
+import io.gatling.core.structure.ScenarioBuilder
+import org.apache.usergrid.enums.ScenarioType
+import org.apache.usergrid.helpers.Setup
+import org.apache.usergrid.scenarios.EntityCollectionScenarios
+import org.apache.usergrid.settings.Settings
+
+import scala.collection.mutable
+
+/**
+ * Simulations with custom injection.
+ */
+class CustomInjectionSimulation extends Simulation {
+
+ def getScenario(scenarioType: String): ScenarioBuilder = {
+ scenarioType match {
+ case ScenarioType.LoadEntities => EntityCollectionScenarios.loadEntities
+ case ScenarioType.DeleteEntities => EntityCollectionScenarios.deleteEntities
+ case ScenarioType.UpdateEntities => EntityCollectionScenarios.updateEntities
+ case ScenarioType.GetAllByCursor => EntityCollectionScenarios.getEntityPagesToEnd
+ case ScenarioType.NameRandomInfinite => EntityCollectionScenarios.getRandomEntitiesByName
+ case ScenarioType.UuidRandomInfinite => EntityCollectionScenarios.getRandomEntitiesByUuid
+ case ScenarioType.GetByNameSequential => EntityCollectionScenarios.getEntitiesByNameSequential
+ case _ => null
+ }
+ }
+
+ before{
+ if (!Settings.skipSetup) {
+ println("Begin setup")
+ if (Settings.createOrg) Setup.setupOrg()
+ if (Settings.createApp) Setup.setupApplication()
+ if (Settings.loadEntities) Setup.setupEntitiesCollection(Settings.numEntities, Settings.entityType, Settings.entityPrefix, Settings.entitySeed)
+ } else {
+ println("Skipping setup")
+ }
+ if (Settings.sandboxCollection) Setup.sandboxCollection()
+ Settings.setTestStartTime()
+ }
+
+ if (ScenarioType.isValid(Settings.scenarioType)) {
+ val scenario: ScenarioBuilder = getScenario(Settings.scenarioType)
+
+ val injectionList:String = Settings.injectionList
+ val injectStepsArray:Array[String] = injectionList.split("\\s*;\\s*")
+ val injectStepList:mutable.ArraySeq[InjectionStep] = new mutable.ArraySeq[InjectionStep](injectStepsArray.length)
+ for (i <- injectStepsArray.indices) {
+ val injectionStep = injectStepsArray(i).trim
+ println(injectionStep)
+ val stepRegex = """(.+)\((.*)\)""".r
+ val stepRegex(stepType,stepArgsStr) = injectionStep
+ println(s"stepType:$stepType stepArgs:$stepArgsStr")
+ val stepArgs = stepArgsStr.split("\\s*,\\s*")
+ injectStepList(i) = stepType match {
+ case "rampUsers" => rampUsers(stepArgs(0).toInt) over stepArgs(1).toInt
+ case "constantUsersPerSec" => constantUsersPerSec(stepArgs(0).toDouble) during stepArgs(1).toInt
+ case "constantUsersPerSecRandomized" => constantUsersPerSec(stepArgs(0).toDouble) during stepArgs(1).toInt randomized
+ case "atOnceUsers" => atOnceUsers(stepArgs(0).toInt)
+ case "rampUsersPerSec" => rampUsersPerSec(stepArgs(0).toDouble) to stepArgs(1).toInt during stepArgs(2).toInt
+ case "rampUsersPerSecRandomized" => rampUsersPerSec(stepArgs(0).toDouble) to stepArgs(1).toInt during stepArgs(2).toInt randomized
+ case "heavisideUsers" => heavisideUsers(stepArgs(0).toInt) over stepArgs(1).toInt
+ case "nothingFor" => nothingFor(stepArgs(0).toInt)
+ }
+ }
+
+ setUp(
+ scenario
+ .inject(injectStepList)
+ .protocols(Settings.httpConf.acceptHeader("application/json"))
+ )
+ } else {
+ println(s"scenarioType ${Settings.scenarioType} not found.")
+ }
+
+ after {
+ endHandler
+ }
+
+ def endHandler: Unit = {
+ Settings.setTestEndTime()
+ if (Settings.captureUuids) Settings.writeUuidsToFile()
+ Settings.printSettingsSummary(true)
+ }
+
+ sys addShutdownHook(endHandler)
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3174e6d/stack/loadtests/testConfig.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/testConfig.sh b/stack/loadtests/testConfig.sh
index 1e20637..0901e99 100755
--- a/stack/loadtests/testConfig.sh
+++ b/stack/loadtests/testConfig.sh
@@ -37,7 +37,7 @@ SANDBOX_COLLECTION=true
NUM_ENTITIES=10000
SKIP_SETUP=false
-COLLECTION=thousand
+COLLECTION=thousands
ENTITY_TYPE=trivialSortable
ENTITY_PREFIX=sortable
ENTITY_SEED=1
@@ -50,8 +50,10 @@ ENTITY_PROGRESS_COUNT=1000
END_CONDITION_TYPE=minutesElapsed
#END_CONDITION_TYPE=requestCount
-END_MINUTES=2
+END_MINUTES=5
END_REQUEST_COUNT=100
CONSTANT_USERS_PER_SEC=0
CONSTANT_USERS_DURATION=10
+
+INJECTION_LIST="rampUsers(100,300);nothingFor(300)"
[2/6] incubator-usergrid git commit: Merge remote-tracking branch
'upstream/two-dot-o-dev' into two-dot-o-dev
Posted by gr...@apache.org.
Merge remote-tracking branch 'upstream/two-dot-o-dev' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dc9dafea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dc9dafea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dc9dafea
Branch: refs/heads/two-dot-o-dev
Commit: dc9dafea53f72fd2125410bc7cf634224b7c9845
Parents: 7253ce5 9427b4c
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Aug 4 16:46:32 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Aug 4 16:46:32 2015 -0700
----------------------------------------------------------------------
stack/awscluster/gatling-cluster-cf.json | 28 +-
.../main/resources/usergrid-default.properties | 4 +
.../corepersistence/ApplicationIdCacheFig.java | 2 +
.../corepersistence/ApplicationIdCacheImpl.java | 14 +-
.../corepersistence/CpEntityManager.java | 35 +-
.../corepersistence/CpRelationManager.java | 6 -
.../corepersistence/EntityManagerFig.java | 4 +-
.../asyncevents/AmazonAsyncEventService.java | 270 ++++++-----
.../asyncevents/AsyncIndexProvider.java | 11 +-
.../asyncevents/EventBuilder.java | 11 +-
.../asyncevents/EventBuilderImpl.java | 21 +-
.../asyncevents/InMemoryAsyncEventService.java | 16 +-
.../asyncevents/model/AsyncEvent.java | 153 +-----
.../asyncevents/model/EdgeDeleteEvent.java | 36 +-
.../asyncevents/model/EdgeIndexEvent.java | 39 +-
.../asyncevents/model/EntityDeleteEvent.java | 18 +-
.../asyncevents/model/EntityIndexEvent.java | 30 +-
.../model/InitializeApplicationIndexEvent.java | 21 +-
.../corepersistence/index/CoreIndexFig.java | 2 +
.../index/IndexProcessorFig.java | 9 +-
.../corepersistence/util/CpEntityMapUtils.java | 284 +-----------
.../org/apache/usergrid/persistence/Query.java | 2 +
.../org/apache/usergrid/persistence/Schema.java | 18 +-
.../persistence/entities/Notification.java | 116 ++++-
.../index/AmazonAsyncEventServiceTest.java | 8 +-
.../index/AsyncIndexServiceTest.java | 14 +-
.../org/apache/usergrid/persistence/GeoIT.java | 6 +-
.../collection/cache/EntityCacheFig.java | 2 +
.../data/DataMigrationManagerImpl.java | 1 +
.../persistence/core/rx/RxSchedulerFig.java | 2 +
.../persistence/core/rx/OrderedMergeTest.java | 57 ++-
.../usergrid/persistence/graph/GraphFig.java | 2 +
stack/corepersistence/model/pom.xml | 7 +
.../model/collection/SchemaManager.java | 35 ++
.../model/entity/EntityToMapConverter.java | 61 +--
.../model/entity/MapToEntityConverter.java | 155 ++++---
.../persistence/model/field/FieldTypeName.java | 1 +
.../model/field/SerializedObjectField.java | 51 ++
stack/corepersistence/pom.xml | 5 +-
.../index/IndexLocationStrategy.java | 4 +-
.../index/impl/EntityMappingParser.java | 14 +-
.../index/impl/EsEntityIndexImpl.java | 17 +-
.../index/impl/EsIndexBufferConsumerImpl.java | 24 +-
.../index/impl/EntityToMapConverterTest.java | 12 +-
stack/corepersistence/queue/pom.xml | 20 +-
.../queue/impl/UsergridAwsCredentials.java | 7 +
.../impl/UsergridAwsCredentialsProvider.java | 5 +-
.../queue/util/AmazonNotificationUtils.java | 230 ++++-----
.../persistence/queue/QueueManagerTest.java | 2 +-
stack/pom.xml | 6 +-
.../rest/applications/ApplicationResource.java | 4 +-
.../rest/applications/ServiceResource.java | 137 ++++--
.../applications/assets/AssetsResource.java | 37 +-
.../rest/applications/users/UsersResource.java | 4 +-
.../AwsPropertiesNotFoundExceptionMapper.java | 37 ++
.../main/resources/usergrid-rest-context.xml | 11 +-
.../applications/assets/AssetResourceIT.java | 23 +-
.../applications/assets/AwsAssetResourceIT.java | 463 +++++++++++++++++++
.../applications/assets/aws/NoAWSCredsRule.java | 127 +++++
.../collection/groups/GroupResourceIT.java | 4 +-
.../collection/users/PermissionsResourceIT.java | 40 +-
.../events/ApplicationRequestCounterIT.java | 14 +-
.../rest/management/RegistrationIT.java | 27 --
.../rest/test/resource/AbstractRestIT.java | 36 +-
.../test/resources/usergrid-test-context.xml | 8 -
stack/rest_integration_tests/config/default.js | 2 +-
stack/rest_integration_tests/lib/connections.js | 7 +-
.../rest_integration_tests/lib/notifications.js | 36 +-
stack/rest_integration_tests/package.json | 7 +-
.../test/notifications/create.js | 2 +-
stack/scripts/migrate_entity_data.py | 315 +++++++++++++
stack/services/pom.xml | 12 +-
.../management/AccountCreationProps.java | 2 +
.../services/AbstractCollectionService.java | 8 +-
.../assets/data/AwsSdkS3BinaryStore.java | 189 ++++++--
.../services/assets/data/BinaryStore.java | 8 +-
.../AwsPropertiesNotFoundException.java | 40 ++
.../notifications/NotificationsService.java | 45 +-
.../impl/ApplicationQueueManagerImpl.java | 2 +-
.../usergrid/services/CollectionServiceIT.java | 5 +-
.../test/resources/usergrid-test-context.xml | 2 +-
81 files changed, 2431 insertions(+), 1121 deletions(-)
----------------------------------------------------------------------
[5/6] incubator-usergrid git commit: Merge remote-tracking branch
'upstream/two-dot-o-dev' into two-dot-o-dev
Posted by gr...@apache.org.
Merge remote-tracking branch 'upstream/two-dot-o-dev' into two-dot-o-dev
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/93fb85af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/93fb85af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/93fb85af
Branch: refs/heads/two-dot-o-dev
Commit: 93fb85afbad6bb7311ae573e3eda60bf51dbcf54
Parents: d3174e6 9f81d7b
Author: Mike Dunker <mi...@calbears.net>
Authored: Wed Aug 12 08:54:25 2015 -0700
Committer: Mike Dunker <mi...@calbears.net>
Committed: Wed Aug 12 08:54:25 2015 -0700
----------------------------------------------------------------------
.../asyncevents/AmazonAsyncEventService.java | 2 -
.../model/InitializeApplicationIndexEvent.java | 15 +-
.../PerformanceEntityRebuildIndexTest.java | 385 --------------
.../usergrid/persistence/RebuildIndexTest.java | 515 +++++++++++++++++++
.../migration/data/DataMigrationManager.java | 12 +-
.../data/DataMigrationManagerImpl.java | 18 +
.../data/DataMigrationManagerImplTest.java | 86 ++++
stack/corepersistence/pom.xml | 2 +-
.../apache/usergrid/rest/MigrateResource.java | 49 +-
9 files changed, 684 insertions(+), 400 deletions(-)
----------------------------------------------------------------------