You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/09 18:17:41 UTC
[1/2] incubator-usergrid git commit: Moved remaining CpEntityManager
timers into the constructor.
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-465 [created] d901d38ca
Moved remaining CpEntityManager timers into the constructor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/45b2a219
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/45b2a219
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/45b2a219
Branch: refs/heads/USERGRID-465
Commit: 45b2a21977b2d62351b679930608753f9b8d030e
Parents: 5e0ce62
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 9 10:03:57 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 9 10:03:57 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 111 ++++++++++++-------
1 file changed, 68 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/45b2a219/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f7a9f8b..3aac61d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -191,8 +191,23 @@ public class CpEntityManager implements EntityManager {
private MetricsFactory metricsFactory;
private Timer aggCounterTimer;
private Timer entCreateTimer;
-
-// /** Short-term cache to keep us from reloading same Entity during single request. */
+ private Timer entCreateBatchTimer;
+ private Timer esDeletePropertyTimer;
+ private Timer entAddDictionaryTimer;
+ private Timer entAddDictionarySetTimer;
+ private Timer entAddDictionaryMapTimer;
+ private Timer entRemoveDictionaryTimer;
+ private Timer entCreateRoleTimer;
+ private Timer entCreateRolePermissionsTimer;
+ private Timer entGrantGroupPermissionTimer;
+ private Timer entRevokeGroupPermissionTimer;
+ private Timer entIncrementAggregateCountersTimer;
+ private Timer entGetAggregateCountersQueryTimer;
+ private Timer entGetEntityCountersTimer;
+ private Timer esIndexEntityCollectionTimer;
+ private Timer entRevokeRolePermissionsTimer;
+
+ // /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -214,10 +229,44 @@ public class CpEntityManager implements EntityManager {
this.cass = this.emf.getCassandraService();
this.counterUtils = this.emf.getCounterUtils();
+
+ //Timer Setup
this.metricsFactory = this.emf.getMetricsFactory();
this.aggCounterTimer =this.metricsFactory.getTimer( CpEntityManager.class,
"cp.entity.get.aggregate.counters.timer" );
this.entCreateTimer =this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.create.timer" );
+ this.entCreateBatchTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.create.batch.timer");
+ this.esDeletePropertyTimer =this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.es.delete.property.timer");
+ this.entAddDictionaryTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.add.dictionary.timer");
+ this.entAddDictionarySetTimer = this.metricsFactory.getTimer( CpEntityManager.class,
+ "cp.entity.add.dictionary.set.timer" );
+ this.entAddDictionaryMapTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.add.dictionary.map.timer");
+ this.entRemoveDictionaryTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.remove.dictionary.timer");
+ this.entCreateRoleTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.create.role.timer");
+ this.entCreateRolePermissionsTimer =this.metricsFactory
+ .getTimer( CpEntityManager.class,
+ "cp.entity.create.role.permissions.timer" );
+ this.entGrantGroupPermissionTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.grant.group.permission.timer");
+ this.entRevokeGroupPermissionTimer = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.revoke.group.permission.timer");
+ this.entIncrementAggregateCountersTimer =this.metricsFactory.getTimer( CpEntityManager.class,
+ "cp.entity.increment.aggregate.counters.timer" );
+ this.entGetAggregateCountersQueryTimer = this.metricsFactory.getTimer( CpEntityManager.class,
+ "cp.entity.get.aggregate.counters.query.timer" );
+ this.entGetEntityCountersTimer = this.metricsFactory.getTimer( CpEntityManager.class,
+ "cp.entity.get.entity.counters.timer" );
+ this.esIndexEntityCollectionTimer = this.metricsFactory
+ .getTimer( CpEntityManager.class, "cp.entity.es.index.entity.to.collection.timer" );
+ this.entRevokeRolePermissionsTimer =
+ this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.revoke.role.permissions.timer");
+
// set to false for now
this.skipAggregateCounters = false;
@@ -347,8 +396,7 @@ public class CpEntityManager implements EntityManager {
A entity = batchCreate( m, entityType, entityClass, properties, importId, timestampUuid );
//Adding graphite metrics
- Timer.Context timeEntityCassCreation = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.create.batch.timer").time();
+ Timer.Context timeEntityCassCreation = entCreateBatchTimer.time();
m.execute();
timeEntityCassCreation.stop();
@@ -999,8 +1047,7 @@ public class CpEntityManager implements EntityManager {
} );
//Adding graphite metrics
- Timer.Context timeESBatch = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.es.delete.property.timer").time();
+ Timer.Context timeESBatch = esDeletePropertyTimer.time();
BetterFuture future = ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
timeESBatch.stop();
// update in all containing collections and connection indexes
@@ -1040,8 +1087,7 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, elementValue, false, timestampUuid );
//Adding graphite metrics
- Timer.Context timeDictionaryCreation = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.add.dictionary.timer").time();
+ Timer.Context timeDictionaryCreation = entAddDictionaryTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeDictionaryCreation.stop();
}
@@ -1065,8 +1111,7 @@ public class CpEntityManager implements EntityManager {
}
//Adding graphite metrics
- Timer.Context timeAddingSetDictionary = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.add.dictionary.set.timer").time();
+ Timer.Context timeAddingSetDictionary = entAddDictionarySetTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeAddingSetDictionary.stop();
}
@@ -1091,8 +1136,7 @@ public class CpEntityManager implements EntityManager {
}
//Adding graphite metrics
- Timer.Context timeMapDictionary = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.add.dictionary.map.timer").time();
+ Timer.Context timeMapDictionary = entAddDictionaryMapTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeMapDictionary.stop();
}
@@ -1272,8 +1316,7 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, true, timestampUuid );
//Adding graphite metrics
- Timer.Context timeRemoveDictionary = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.remove.dictionary.timer").time();
+ Timer.Context timeRemoveDictionary = entRemoveDictionaryTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRemoveDictionary.stop();
@@ -1612,8 +1655,7 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), Schema.DICTIONARY_ROLENAMES, null,
timestamp );
//Adding graphite metrics
- Timer.Context timeCreateBatchRole= this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.create.role.timer").time();
+ Timer.Context timeCreateBatchRole= entCreateRoleTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeCreateBatchRole.stop();
@@ -1649,9 +1691,7 @@ public class CpEntityManager implements EntityManager {
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp);
}
//Adding graphite metrics
- Timer.Context timeGrantRolePermissions = this.metricsFactory
- .getTimer( CpEntityManager.class,
- "cp.entity.create.role.permissions.timer" ).time();
+ Timer.Context timeGrantRolePermissions = entCreateRolePermissionsTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGrantRolePermissions.stop();
@@ -1685,8 +1725,7 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, timestamp );
//Adding graphite metrics
- Timer.Context timeRevokeRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.revoke.role.permissions.timer").time();
+ Timer.Context timeRevokeRolePermission = entRevokeRolePermissionsTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRevokeRolePermission.stop();
}
@@ -1754,8 +1793,7 @@ public class CpEntityManager implements EntityManager {
getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
- Timer.Context timeGroupRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.grant.group.permission.timer").time();
+ Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGroupRolePermission.stop();
}
@@ -1770,8 +1808,7 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( groupId, roleName ), permission, timestamp );
//Adding graphite metrics
- Timer.Context timeRevokeGroupRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
- "cp.entity.revoke.group.permission.timer").time();
+ Timer.Context timeRevokeGroupRolePermission = entRevokeGroupPermissionTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRevokeGroupRolePermission.stop();
}
@@ -1923,10 +1960,7 @@ public class CpEntityManager implements EntityManager {
category, counterName, value, cassandraTimestamp / 1000, cassandraTimestamp );
//Adding graphite metrics
- Timer.Context timeIncrementAggregateCounters = this.metricsFactory
- .getTimer( CpEntityManager.class,
- "cp.entity.increment.aggregate.counters.timer" )
- .time();
+ Timer.Context timeIncrementAggregateCounters = entIncrementAggregateCountersTimer.time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
timeIncrementAggregateCounters.stop();
}
@@ -2029,9 +2063,7 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( APPLICATION_AGGREGATE_COUNTERS.toString() );
q.setRange( start, finish, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeGetAggregateCounters = this.metricsFactory
- .getTimer( CpEntityManager.class,
- "cp.entity.get.aggregate.counters.query.timer" ).time();
+ Timer.Context timeGetAggregateCounters = entGetAggregateCountersQueryTimer.time();
QueryResult<CounterRows<String, Long>> rows = q.setKeys( selections.keySet() ).execute();
timeGetAggregateCounters.stop();
@@ -2158,9 +2190,7 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( ENTITY_COUNTERS.toString() );
q.setRange( null, null, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeEntityCounters = this.metricsFactory
- .getTimer( CpEntityManager.class,
- "cp.entity.get.entity.counters.timer" ).time();
+ Timer.Context timeEntityCounters = entGetEntityCountersTimer.time();
QueryResult<CounterSlice<String>> r = q.setKey( entityId ).execute();
timeEntityCounters.stop();
for ( HCounterColumn<String> column : r.get().getColumns() ) {
@@ -2183,13 +2213,10 @@ public class CpEntityManager implements EntityManager {
if ( !skipAggregateCounters ) {
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> m = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- counterUtils.batchIncrementAggregateCounters(
- m, applicationId, userId, groupId, null, category, counters, timestamp );
+ counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId, null, category, counters, timestamp );
//Adding graphite metrics
- Timer.Context timeIncrementCounters = this.metricsFactory
- .getTimer( CpEntityManager.class,
- "cp.entity.increment.aggregate.counters.timer" ).time();
+ Timer.Context timeIncrementCounters =entIncrementAggregateCountersTimer.time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
timeIncrementCounters.stop();
}
@@ -2923,9 +2950,7 @@ public class CpEntityManager implements EntityManager {
// batch.index(appAllTypesScope, memberEntity);
//Adding graphite metrics
- Timer.Context timeIndexEntityCollection = this.metricsFactory
- .getTimer( CpEntityManager.class,
- "cp.entity.es.index.entity.to.collection.timer" ).time();
+ Timer.Context timeIndexEntityCollection = esIndexEntityCollectionTimer.time();
batch.execute();
timeIndexEntityCollection.stop();
}
[2/2] incubator-usergrid git commit: Merge branch 'two-dot-o' of
https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-465
Posted by gr...@apache.org.
Merge branch 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into USERGRID-465
# By Todd Nine (12) and Shawn Feldman (5)
# Via Todd Nine (4) and Shawn Feldman (2)
* 'two-dot-o' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid:
Changed log level to warn to catch issues with version mismatch in logging
Changes index declaration to be explicit on the index
Added operational logging to tasks
Moved add alias and test new index into try so we don't keep moving the index alias
Fixes merge error
Adds put test to gatling
Fixes lookup cache bug
changing buffer impl to queue
changing buffer impl to queue
Added caching to org/app lookup
Upgraded plexus utils and fixes build concurrency issues with surefire
windows raw notifications client side
windows push raw notifications
Updated tomcat threads to be configurable per core in cloud formation template
Revert "Merge branch 'context-term-hotfix' into two-dot-o"
Changes the search for context from term to match, since it should be an exact match query.
remove createindex calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d901d38c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d901d38c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d901d38c
Branch: refs/heads/USERGRID-465
Commit: d901d38caf68c00e12e7a3271f82255e7c02fa94
Parents: 45b2a21 673c057
Author: GERey <gr...@apigee.com>
Authored: Mon Mar 9 10:04:21 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Mon Mar 9 10:04:21 2015 -0700
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 +++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 ++-
.../main/dist/init_instance/init_rest_server.sh | 3 -
.../src/main/groovy/configure_usergrid.groovy | 6 +-
stack/awscluster/ugcluster-cf.json | 8 +
stack/core/pom.xml | 7 +-
.../corepersistence/CpEntityManager.java | 1 +
.../corepersistence/CpEntityManagerFactory.java | 147 +++++----------
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++++
.../corepersistence/results/EntityVerifier.java | 4 +-
.../results/FilteringLoader.java | 2 +-
.../main/resources/usergrid-core-context.xml | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../core/metrics/MetricsFactoryImpl.java | 3 +-
.../persistence/index/EntityIndexFactory.java | 4 +-
.../persistence/index/IndexBufferProducer.java | 10 +-
.../usergrid/persistence/index/IndexFig.java | 12 +-
.../persistence/index/guice/IndexModule.java | 7 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 79 ++++++++
.../index/impl/EsEntityIndexImpl.java | 85 ++++++---
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++-
.../persistence/index/impl/IndexingUtils.java | 5 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../datagenerators/EntityDataGenerator.scala | 25 +++
.../datagenerators/FeederGenerator.scala | 18 +-
.../usergrid/scenarios/EntityScenarios.scala | 8 +-
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 ++++++++
stack/pom.xml | 24 ++-
stack/rest/pom.xml | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 19 ++
stack/services/pom.xml | 7 +-
.../services/notifications/wns/WNSAdapter.java | 39 ++++
.../resources/usergrid-services-context.xml | 8 +-
.../usergrid/management/OrganizationIT.java | 2 +-
39 files changed, 799 insertions(+), 205 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d901d38c/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------