You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2015/10/06 23:25:19 UTC
usergrid git commit: USERGRID-1044: support for multi-worker tests
without end that can be audited
Repository: usergrid
Updated Branches:
refs/heads/USERGRID-1044 91fa48940 -> a9927a3ba
USERGRID-1044: support for multi-worker tests without end
that can be audited
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a9927a3b
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a9927a3b
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a9927a3b
Branch: refs/heads/USERGRID-1044
Commit: a9927a3baa23bcd30dcacb4255af649c38ed3d90
Parents: 91fa489
Author: Mike Dunker <md...@apigee.com>
Authored: Tue Oct 6 14:24:40 2015 -0700
Committer: Mike Dunker <md...@apigee.com>
Committed: Tue Oct 6 14:24:40 2015 -0700
----------------------------------------------------------------------
stack/loadtests/runDeleteEntities.sh | 13 ++-
stack/loadtests/runLoadEntitiesUnlimited.sh | 101 +++++++++++++++++++
.../datagenerators/FeederGenerator.scala | 9 +-
.../usergrid/enums/ConfigProperties.scala | 11 +-
.../apache/usergrid/helpers/Extractors.scala | 10 ++
.../scenarios/EntityCollectionScenarios.scala | 27 +++--
.../org/apache/usergrid/settings/Settings.scala | 94 ++++++++++++-----
stack/loadtests/testConfig.sh | 2 +
8 files changed, 227 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/runDeleteEntities.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runDeleteEntities.sh b/stack/loadtests/runDeleteEntities.sh
index 2013e89..d0da19b 100755
--- a/stack/loadtests/runDeleteEntities.sh
+++ b/stack/loadtests/runDeleteEntities.sh
@@ -42,24 +42,27 @@ if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
die() { echo "$@" 1>&2 ; exit 1; }
-[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
+[ "$#" -ge 3 ] || die "At least 3 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) ENTITY_PREFIX [NUM_ENTITIES [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
RAMP_USERS="$1"
RAMP_TIME="$2"
-[ "$#" -ge 3 ] && NUM_ENTITIES="$3"
-[ "$#" -ge 4 ] && ENTITY_SEED="$4"
-[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
-[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+ENTITY_PREFIX="$3"
+[ "$#" -ge 4 ] && NUM_ENTITIES="$4"
+[ "$#" -ge 5 ] && ENTITY_SEED="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_NUM="$6"
+[ "$#" -ge 7 ] && ENTITY_WORKER_COUNT="$7"
shift $#
SCENARIO_TYPE=deleteEntities
+INTERLEAVED_WORKER_FEED=true
#Compile everything
mvn compile
#Execute the test
mvn gatling:execute \
+-DinterleavedWorkerFeed=${INTERLEAVED_WORKER_FEED} \
-DbaseUrl=${URL} \
-DadminUser=${ADMIN_USER} \
-DadminPassword=${ADMIN_PASSWORD} \
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/runLoadEntitiesUnlimited.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/runLoadEntitiesUnlimited.sh b/stack/loadtests/runLoadEntitiesUnlimited.sh
new file mode 100755
index 0000000..9222a57
--- /dev/null
+++ b/stack/loadtests/runLoadEntitiesUnlimited.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DIR="${BASH_SOURCE%/*}"
+if [[ ! -d "$DIR" ]]; then DIR="$PWD"; fi
+. "$DIR/testConfig.sh"
+
+# from testConfig.sh
+#URL=
+#ADMIN_USER=
+#ADMIN_PASSWORD=
+#ENTITY_WORKER_NUM= #may be overridden on command line
+#ENTITY_WORKER_COUNT= #may be overridden on command line
+#ORG=
+#APP=
+#AUTH_TYPE=
+#TOKEN_TYPE=
+#CREATE_ORG=
+#CREATE_APP=
+#SANDBOX_COLLECTION=
+#NUM_ENTITIES= #ignored
+#SKIP_SETUP=
+#COLLECTION=
+#ENTITY_TYPE=
+#ENTITY_PREFIX=
+#ENTITY_SEED= #may be overridden on command line
+#RETRY_COUNT=
+#ENTITY_PROGRESS_COUNT=
+#CONSTANT_USERS_PER_SEC=
+#CONSTANT_USERS_DURATION=
+#FLUSH_CSV
+
+die() { echo "$@" 1>&2 ; exit 1; }
+
+[ "$#" -ge 2 ] || die "At least 2 arguments required, $# provided. Example is $0 RAMP_USERS RAMP_TIME(seconds) [UUID_FILENAME [ENTITY_SEED [ENTITY_WORKER_NUM [ENTITY_WORKER_COUNT]]]]"
+
+RAMP_USERS="$1"
+RAMP_TIME="$2"
+[ "$#" -ge 3 ] && UUID_FILENAME="$3"
+[ "$#" -ge 4 ] && ENTITY_SEED="$4"
+[ "$#" -ge 5 ] && ENTITY_WORKER_NUM="$5"
+[ "$#" -ge 6 ] && ENTITY_WORKER_COUNT="$6"
+
+shift $#
+
+SCENARIO_TYPE=loadEntities
+NEW_CSV_ON_FLUSH=true
+FLUSH_CSV=10000
+
+# don't load entities as part of setup (loading entities is the point of the test)
+LOAD_ENTITIES=false
+
+#Compile everything
+mvn compile
+
+#Execute the test
+mvn gatling:execute \
+-DflushCsv=${FLUSH_CSV} \
+-DunlimitedFeed=true \
+-DnewCsvOnFlush=${NEW_CSV_ON_FLUSH} \
+-DbaseUrl=${URL} \
+-DadminUser=${ADMIN_USER} \
+-DadminPassword=${ADMIN_PASSWORD} \
+-DentityWorkerNum=${ENTITY_WORKER_NUM} \
+-DentityWorkerCount=${ENTITY_WORKER_COUNT} \
+-Dorg=${ORG} \
+-Dapp=${APP} \
+-DauthType=${AUTH_TYPE} \
+-DtokenType=${TOKEN_TYPE} \
+-DcreateOrg=${CREATE_ORG} \
+-DcreateApp=${CREATE_APP} \
+-DsandboxCollection=${SANDBOX_COLLECTION} \
+-DskipSetup=${SKIP_SETUP} \
+-Dcollection=${COLLECTION} \
+-DentityType=${ENTITY_TYPE} \
+-DentityPrefix=${ENTITY_PREFIX} \
+-DentitySeed=${ENTITY_SEED} \
+-DretryCount=${RETRY_COUNT} \
+-DentityProgressCount=${ENTITY_PROGRESS_COUNT} \
+-DconstantUsersPerSec=${CONSTANT_USERS_PER_SEC} \
+-DconstantUsersDuration=${CONSTANT_USERS_DURATION} \
+-DscenarioType=${SCENARIO_TYPE} \
+-DloadEntities=${LOAD_ENTITIES} \
+-DrampUsers=${RAMP_USERS} \
+-DrampTime=${RAMP_TIME} \
+-DuuidFilename=${UUID_FILENAME} \
+-DprintFailedRequests=${PRINT_FAILED_REQUESTS} \
+-Dgatling.simulationClass=org.apache.usergrid.simulations.ConfigurableSimulation
+
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index ca93fdc..e08f8e8 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -220,16 +220,19 @@ object FeederGenerator {
override def next(): Map[String, String] = {
val i = counter.getAndIncrement()
- val seededVal = i + seed
+ val seededVal = if (Settings.interleavedWorkerFeed) seed + (i * Settings.entityWorkerCount) + (Settings.entityWorkerNum - 1) else i + seed
val noPrefix = prefix == null || prefix == ""
val entityName = if (noPrefix) seededVal.toString else prefix.concat(seededVal.toString)
val entity = EntityDataGenerator.generateEntity(entityType, if (noPrefix) null else entityName, seededVal)
+ //println(entity)
val entityUrl = Settings.baseCollectionUrl + "/" + entityName
- val validEntity = if (i >= numEntities) "no" else "yes"
+ val validEntity = if (!Settings.unlimitedFeed && i >= numEntities) "no" else "yes"
+ val collectionName = Settings.app + "/" + Settings.collection
// println(entityName)
- Map("entityName" -> entityName, "entity" -> entity, "entityUrl" -> entityUrl, "validEntity" -> validEntity, "entityNum" -> (i+1).toString, "seededEntityNum" -> seededVal.toString)
+ Map("entityName" -> entityName, "entity" -> entity, "entityUrl" -> entityUrl, "validEntity" -> validEntity, "entityNum" -> (i+1).toString, "seededEntityNum" -> seededVal.toString,
+ "collectionName" -> collectionName)
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
index df0b325..ba09979 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/enums/ConfigProperties.scala
@@ -81,6 +81,10 @@ object ConfigProperties {
val EntityNumberProperty = "entityNumberProperty"
val QueryParams = "queryParams"
val CsvFeedPattern = "csvFeedPattern"
+ val UnlimitedFeed = "unlimitedFeed"
+ val FlushCsv = "flushCsv"
+ val InterleavedWorkerFeed = "interleavedWorkerFeed"
+ val NewCsvOnFlush = "newCsvOnFlush"
val Values = Seq(Org,App,AdminUser,AdminPassword,BaseUrl,AuthType,TokenType,SkipSetup,CreateOrg,CreateApp,LoadEntities,
ScenarioType,RampUsers,ConstantUsersPerSec,ConstantUsersDuration,UserSeed,AppUser,AppUserPassword,NumEntities,
@@ -89,7 +93,8 @@ object ConfigProperties {
OrgCreationName,OrgCreationEmail,OrgCreationPassword,UpdateProperty,UpdateValue,EntityWorkerCount,EntityWorkerNum,
UuidFilename,AuditUuidFilename,FailedUuidFilename,SandboxCollection,PurgeUsers,RetryCount,LaterThanTimestamp,
EntityProgressCount,InjectionList,PrintFailedRequests,GetViaQuery,MultiPropertyPrefix,MultiPropertyCount,
- MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern)
+ MultiPropertySizeInK,EntityNumberProperty,QueryParams,CsvFeedPattern,UnlimitedFeed,FlushCsv,InterleavedWorkerFeed,
+ NewCsvOnFlush)
def isValid(str: String): Boolean = {
Values.contains(str)
@@ -158,6 +163,10 @@ object ConfigProperties {
case EntityNumberProperty => ""
case QueryParams => ""
case CsvFeedPattern => org.apache.usergrid.enums.CsvFeedPatternType.Random
+ case UnlimitedFeed => false
+ case FlushCsv => 0L
+ case InterleavedWorkerFeed => false
+ case NewCsvOnFlush => false
}
} else {
null
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
index d93c430..8a18053 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/helpers/Extractors.scala
@@ -51,6 +51,16 @@ object Extractors {
}
/**
+ * Will extract the modified date from the create response. If the modified field is not present, -1 will be set
+ */
+ def extractCreateModified(saveAsName: String) = {
+ jsonPath("$.entities[0].modified").ofType[Long].transformOption(extract => {
+ //it may or may not be present. If it is, save it, otherwise save it as -1
+ extract.orElse(Some(-1))
+ }).saveAs(saveAsName)
+ }
+
+ /**
* Will extract the audit entities from the get collection response.
*/
def extractAuditEntities(saveAsName: String) = {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
index 0ab2cd6..1b15103 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityCollectionScenarios.scala
@@ -36,6 +36,7 @@ object EntityCollectionScenarios {
//The value for the cursor
val SessionVarCursor: String = "cursor"
val SessionVarUuid: String = "createUuid"
+ val SessionVarModified: String = "createModified"
def entityGetUrl(useCursor: Boolean): String = {
val url = s"/${Settings.collection}?" +
@@ -208,9 +209,13 @@ object EntityCollectionScenarios {
.headers(Headers.authToken)
.body(StringBody("""${entity}"""))
// 200 for success, 400 if already exists
- .check(status.in(Seq(200)), extractCreateUuid(SessionVarUuid)))
+ .check(status.in(Seq(200)), extractCreateUuid(SessionVarUuid), extractCreateModified(SessionVarModified)))
.exec(session => {
- Settings.addUuid(session("seededEntityNum").as[String].toInt, session(SessionVarUuid).as[String])
+ val uuid = session(SessionVarUuid).as[String]
+ val entityName = session("entityName").as[String]
+ val modified = session(SessionVarModified).as[Long]
+ val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified)
session
})
}
@@ -294,9 +299,13 @@ object EntityCollectionScenarios {
.get("/" + Settings.collection + "/${entityName}")
.queryParamMap(Settings.queryParamMap)
.headers(Headers.authAnonymous)
- .check(status.is(200), extractCreateUuid(SessionVarUuid)))
+ .check(status.is(200), extractCreateUuid(SessionVarUuid), extractCreateModified(SessionVarModified)))
.exec(session => {
- Settings.addUuid(session("seededEntityNum").as[String].toInt, session(SessionVarUuid).as[String])
+ val uuid = session(SessionVarUuid).as[String]
+ val entityName = session("entityName").as[String]
+ val modified = session(SessionVarModified).as[Long]
+ val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified)
session
})
}
@@ -308,10 +317,14 @@ object EntityCollectionScenarios {
.get("/" + Settings.collection + "/${entityName}")
.queryParamMap(Settings.queryParamMap)
.headers(Headers.authToken)
- .check(status.is(200), extractCreateUuid(SessionVarUuid)))
+ .check(status.is(200), extractCreateUuid(SessionVarUuid), extractCreateModified(SessionVarModified)))
.exec(session => {
- Settings.addUuid(session("seededEntityNum").as[String].toInt, session(SessionVarUuid).as[String])
- session
+ val uuid = session(SessionVarUuid).as[String]
+ val entityName = session("entityName").as[String]
+ val modified = session(SessionVarModified).as[Long]
+ val collectionName = session("collectionName").as[String]
+ Settings.addUuid(uuid, collectionName, entityName, modified)
+ session
})
}
)
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index e9dd2f2..27c1d33 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -181,6 +181,11 @@ object Settings {
private val queryParamConfig = initStrSetting(ConfigProperties.QueryParams)
val queryParamMap: Map[String,String] = mapFromQueryParamConfigString(queryParamConfig)
val csvFeedPattern = initStrSetting(ConfigProperties.CsvFeedPattern)
+ val flushCsv:Long = initLongSetting(ConfigProperties.FlushCsv)
+ val unlimitedFeed:Boolean = initBoolSetting(ConfigProperties.UnlimitedFeed)
+ // unlimited feed forces interleaved worker feed
+ val interleavedWorkerFeed:Boolean = if (unlimitedFeed) true else initBoolSetting(ConfigProperties.InterleavedWorkerFeed)
+ val newCsvOnFlush:Boolean = initBoolSetting(ConfigProperties.NewCsvOnFlush)
val multiPropertyPrefix = initStrSetting(ConfigProperties.MultiPropertyPrefix)
val multiPropertyCount:Int = initIntSetting(ConfigProperties.MultiPropertyCount)
@@ -204,8 +209,9 @@ object Settings {
private val leftOver:Int = totalNumEntities % entityWorkerCount // will be 0 if only one worker
private val extraEntity:Int = if (entityWorkerNum <= leftOver) 1 else 0
private val zeroBasedWorkerNum:Int = entityWorkerNum - 1
- val entitySeed:Int = overallEntitySeed + zeroBasedWorkerNum * entitiesPerWorkerFloor + (if (extraEntity == 1) zeroBasedWorkerNum else leftOver)
- val numEntities:Int = entitiesPerWorkerFloor + extraEntity
+ val entitySeed:Int = if (unlimitedFeed) overallEntitySeed else overallEntitySeed + zeroBasedWorkerNum * entitiesPerWorkerFloor + (if (extraEntity == 1) zeroBasedWorkerNum else leftOver)
+ // numEntities is used for random name generation, must be >= 0 even if not used for entity counting (as in unlimitedFeed=true)
+ val numEntities:Int = if (unlimitedFeed) 1000000000 else entitiesPerWorkerFloor + extraEntity
// UUID log file, have to go through this because creating a csv feeder with an invalid csv file fails at maven compile time
private val dummyTestCsv = ConfigProperties.getDefault(ConfigProperties.UuidFilename).toString
@@ -281,18 +287,53 @@ object Settings {
val purgeUsers:Int = initIntSetting(ConfigProperties.PurgeUsers)
- private var uuidMap: Map[Int, String] = Map()
- private var entityCounter: Long = 0
- private var lastEntityCountPrinted: Long = 0
- def addUuid(num: Int, uuid: String): Unit = {
+ val auditUuidsHeader = "collection,name,uuid,modified"
+ val uuidsHeader = "name,uuid"
+ case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long)
+
+ //private var uuidMap: Map[Int, String] = Map()
+ private var uuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
+ private var entityCounter: Long = 0L
+ private var lastEntityCountPrinted: Long = 0L
+ private var flushCounter: Long = 0L
+ private var firstFlush: Boolean = true
+ private var numberFlushes: Long = 0L
+ private var uuidWriter: PrintWriter = null
+ def addUuid(uuid: String, collection: String, entityName: String, modified: Long): Unit = {
if (captureUuids) {
- uuidMap.synchronized {
- uuidMap += (num -> uuid)
- entityCounter += 1
+ uuidList.synchronized {
+ uuidList += AuditList(collection, entityName, uuid, modified)
+ entityCounter += 1L
+ flushCounter += 1L
if (logEntityProgress && (entityCounter >= lastEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $entityCounter")
lastEntityCountPrinted = entityCounter
}
+ if (flushCsv > 0 && flushCounter >= flushCsv) {
+ if (uuidWriter == null) {
+ uuidWriter = {
+ val fileWithSuffix = f"$captureUuidFilename.$numberFlushes%04d"
+ val fos = new FileOutputStream(if (newCsvOnFlush) fileWithSuffix else captureUuidFilename)
+ new PrintWriter(fos, false)
+ }
+ }
+ if (newCsvOnFlush || firstFlush) {
+ uuidWriter.println(auditUuidsHeader)
+ }
+ val sortedUuidList: List[AuditList] = uuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified))
+ sortedUuidList.foreach { e =>
+ uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+ }
+ uuidWriter.flush()
+ if (newCsvOnFlush) {
+ uuidWriter.close()
+ uuidWriter = null
+ }
+ flushCounter = 0L
+ numberFlushes += 1L
+ uuidList.clear()
+ firstFlush = false
+ }
}
}
// println(s"UUID: ${name},${uuid}")
@@ -300,34 +341,39 @@ object Settings {
def writeUuidsToFile(): Unit = {
if (captureUuids) {
- val writer = {
- val fos = new FileOutputStream(captureUuidFilename)
- new PrintWriter(fos, false)
+ if (uuidWriter == null) {
+ uuidWriter = {
+ val fileWithSuffix = f"$captureUuidFilename.$numberFlushes%04d"
+ val fos = new FileOutputStream(if (newCsvOnFlush) fileWithSuffix else captureUuidFilename)
+ new PrintWriter(fos, false)
+ }
}
- writer.println("name,uuid")
- val uuidList: List[(Int, String)] = uuidMap.toList.sortBy(l => l._1)
- uuidList.foreach { l =>
- writer.println(s"${Settings.entityPrefix}${l._1},${l._2}")
+ if (newCsvOnFlush || firstFlush) {
+ uuidWriter.println(auditUuidsHeader)
}
- writer.flush()
- writer.close()
+ val sortedUuidList: List[AuditList] = uuidList.toList.sortBy(e => (e.collection, e.entityName, e.modified))
+ sortedUuidList.foreach { e =>
+ uuidWriter.println(s"${e.collection},${e.entityName},${e.uuid},${e.modified}")
+ }
+ uuidWriter.flush()
+ uuidWriter.close()
+ numberFlushes += 1L
+ uuidList.clear()
+ firstFlush = false
}
}
- val auditUuidsHeader = "collection,name,uuid,modified"
-
- case class AuditList(var collection: String, var entityName: String, var uuid: String, var modified: Long)
// key: uuid, value: collection
- private var auditEntityCounter: Long = 0
- private var lastAuditEntityCountPrinted: Long = 0
+ private var auditEntityCounter: Long = 0L
+ private var lastAuditEntityCountPrinted: Long = 0L
private var auditUuidList: mutable.MutableList[AuditList] = mutable.MutableList[AuditList]()
def addAuditUuid(uuid: String, collection: String, entityName: String, modified: Long): Unit = {
if (captureAuditUuids) {
auditUuidList.synchronized {
auditUuidList += AuditList(collection, entityName, uuid, modified)
- auditEntityCounter += 1
+ auditEntityCounter += 1L
if (logEntityProgress && (auditEntityCounter >= lastAuditEntityCountPrinted + entityProgressCount)) {
println(s"Entity: $auditEntityCounter")
lastAuditEntityCountPrinted = auditEntityCounter
http://git-wip-us.apache.org/repos/asf/usergrid/blob/a9927a3b/stack/loadtests/testConfig.sh
----------------------------------------------------------------------
diff --git a/stack/loadtests/testConfig.sh b/stack/loadtests/testConfig.sh
index 72b17c3..466d8c9 100755
--- a/stack/loadtests/testConfig.sh
+++ b/stack/loadtests/testConfig.sh
@@ -62,3 +62,5 @@ PRINT_FAILED_REQUESTS=true
GET_VIA_QUERY=false
QUERY_PARAMS=
+
+FLUSH_CSV=10000