You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/09 16:25:14 UTC
[01/31] incubator-usergrid git commit: remove createindex calls
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-405 0c3b4395e -> b6351134e
remove createindex calls
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0c15e9b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0c15e9b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0c15e9b1
Branch: refs/heads/USERGRID-405
Commit: 0c15e9b1241102e23ff9377aeaaf966ee49ed417
Parents: ccc884b
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 4 10:32:33 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 4 10:32:33 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 25 ++--------------
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../index/impl/EsEntityIndexImpl.java | 31 +++++++++++++-------
3 files changed, 23 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c15e9b1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9aa36a0..7a913b0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -86,10 +86,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
/** Have we already initialized the index for the management app? */
private AtomicBoolean indexInitialized = new AtomicBoolean( );
- /** Keep track of applications that already have indexes to avoid redundant re-creation. */
- private static final Set<UUID> applicationIndexesCreated = new HashSet<UUID>();
-
-
// cache of already instantiated entity managers
private LoadingCache<UUID, EntityManager> entityManagers
= CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
@@ -156,18 +152,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
-// public ManagerCache getManagerCache() {
-//
-// if ( managerCache == null ) {
-// managerCache = injector.getInstance( ManagerCache.class );
-//
-// dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-// }
-// return managerCache;
-// }
-
-
-
@Override
public EntityManager getEntityManager(UUID applicationId) {
try {
@@ -185,12 +169,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityManager em = new CpEntityManager();
em.init( this, applicationId );
- // only need to do this once
- if ( !applicationIndexesCreated.contains( applicationId ) ) {
- em.createIndex();
- applicationIndexesCreated.add( applicationId );
- }
-
return em;
}
@@ -232,7 +210,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public UUID initializeApplication( String organizationName, UUID applicationId, String name,
Map<String, Object> properties ) throws Exception {
-
EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
final String appName = buildAppName( organizationName, name );
@@ -279,6 +256,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}catch(DuplicateUniquePropertyExistsException e){
throw new ApplicationAlreadyExistsException( appName );
}
+ em.createIndex();
em.refreshIndex();
// create application entity
@@ -289,6 +267,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityManager appEm = getEntityManager( applicationId );
appEm.create( applicationId, TYPE_APPLICATION, properties );
+ appEm.createIndex();
appEm.resetRoles();
appEm.refreshIndex();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c15e9b1/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index 9392d06..dea4967 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -52,6 +52,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
+@NotThreadSafe
public class EntityManagerFactoryImplIT extends AbstractCoreIT {
private static final Logger logger = LoggerFactory.getLogger( EntityManagerFactoryImplIT.class );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0c15e9b1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 49026e2..eac3822 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -61,6 +61,7 @@ import org.elasticsearch.index.query.*;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.indices.InvalidAliasNameException;
+import org.elasticsearch.rest.action.admin.indices.alias.delete.AliasesMissingException;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.sort.FieldSortBuilder;
@@ -192,11 +193,19 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
String[] indexNames = getIndexes(AliasType.Write);
- for (String currentIndex : indexNames){
- isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
+ for (String currentIndex : indexNames) {
+ try {
+ isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
alias.getWriteAlias()).execute().actionGet().isAcknowledged();
- logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck);
+ logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck);
+ } catch (AliasesMissingException aie) {
+ logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, aie.getMessage());
+ continue;
+ }catch(InvalidAliasNameException iane) {
+ logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, iane.getMessage());
+ continue;
+ }
}
// add read alias
@@ -218,7 +227,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public String[] getIndexes(final AliasType aliasType) {
- return aliasCache.getIndexes(alias,aliasType);
+ return aliasCache.getIndexes(alias, aliasType);
}
@@ -270,7 +279,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private void createMappings() throws IOException {
XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_");
+ XContentFactory.jsonBuilder(), "_default_");
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
@@ -299,7 +308,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final String context = IndexingUtils.createContextName(indexScope);
final String[] entityTypes = searchTypes.getTypeNames();
- QueryBuilder qb = query.createQueryBuilder( context );
+ QueryBuilder qb = query.createQueryBuilder(context);
SearchResponse searchResponse;
@@ -410,7 +419,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final SearchHit[] hits = searchHits.getHits();
final int length = hits.length;
- logger.debug( " Hit count: {} Total hits: {}", length, searchHits.getTotalHits() );
+ logger.debug(" Hit count: {} Total hits: {}", length, searchHits.getTotalHits());
List<CandidateResult> candidates = new ArrayList<>( length );
@@ -488,7 +497,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//since we don't have paging inputs, there's no point in executing a query for paging.
final String context = IndexingUtils.createContextName(scope);
- final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(id.getType());
final QueryBuilder queryBuilder =
QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
@@ -520,7 +529,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
String idString = IndexingUtils.idString(entityId).toLowerCase();
- final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
+ final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
@@ -528,10 +537,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
response.addListener(new ActionListener<DeleteByQueryResponse>() {
@Override
public void onResponse(DeleteByQueryResponse response) {
- logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
+ logger.debug("Deleted entity {}:{} from all index scopes with response status = {}",
entityId.getType(), entityId.getUuid(), response.status().toString());
- checkDeleteByQueryResponse( tqb, response );
+ checkDeleteByQueryResponse(tqb, response);
}
@Override
[27/31] incubator-usergrid git commit: Added operational logging to
tasks
Posted by sf...@apache.org.
Added operational logging to tasks
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5decb0db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5decb0db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5decb0db
Branch: refs/heads/USERGRID-405
Commit: 5decb0db1639271c1e8b2a77be373f97334df26c
Parents: fc07d9a
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 8 21:49:10 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 8 21:49:10 2015 -0600
----------------------------------------------------------------------
.../org/apache/usergrid/rest/IndexResource.java | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5decb0db/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
index 3269ec0..2ffc13f 100644
--- a/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
+++ b/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
@@ -89,6 +89,8 @@ public class IndexResource extends AbstractContextResource {
catch ( Exception e ) {
logger.error( "Unable to rebuild indexes", e );
}
+
+ logger.info( "Completed all indexes" );
}
};
@@ -131,12 +133,20 @@ public class IndexResource extends AbstractContextResource {
@Override
public void run() {
+
+
+ logger.info( "Started rebuilding application {} in collection ", appId );
+
+
try {
emf.rebuildApplicationIndexes( appId, po );
}
catch ( Exception e ) {
logger.error( "Unable to re-index application", e );
}
+
+
+ logger.info( "Completed rebuilding application {} in collection ", appId );
}
};
@@ -168,6 +178,8 @@ public class IndexResource extends AbstractContextResource {
public void run() {
+ logger.info( "Started rebuilding application {} in collection {}", appId, collectionName );
+
try {
rebuildCollection( appId, collectionName, reverse );
} catch (Exception e) {
@@ -175,6 +187,8 @@ public class IndexResource extends AbstractContextResource {
// TODO: handle this in rebuildCollection() instead
throw new RuntimeException("Error rebuilding collection");
}
+
+ logger.info( "Completed rebuilding application {} in collection {}", appId, collectionName );
}
};
@@ -214,12 +228,17 @@ public class IndexResource extends AbstractContextResource {
@Override
public void run() {
+
+ logger.info( "Started rebuilding internal indexes", appId );
+
try {
emf.rebuildInternalIndexes( po );
}
catch ( Exception e ) {
logger.error( "Unable to re-index internals", e );
}
+
+ logger.info( "Completed rebuilding internal indexes" );
}
};
[17/31] incubator-usergrid git commit: changing buffer impl to queue
Posted by sf...@apache.org.
changing buffer impl to queue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6066b6e9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6066b6e9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6066b6e9
Branch: refs/heads/USERGRID-405
Commit: 6066b6e98e3f3324858d8812fb607bf77b37f9a8
Parents: 5ad1a8c
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 15:50:48 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 15:50:48 2015 -0700
----------------------------------------------------------------------
.../notifications/Usergrid.Notifications/MainPage.xaml.cs | 8 ++++----
.../persistence/index/impl/EsIndexBufferConsumerImpl.java | 2 --
2 files changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6066b6e9/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
index ccef239..2120eeb 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
@@ -44,7 +44,7 @@ namespace Usergrid.Notifications
{
this.InitializeComponent();
//TODO: change me to your server
- serverUrl = "http://10.0.1.20:8080/";
+ serverUrl = "http://server/";
//TODO: change me to your org
org = "test-organization";
//TODO: change me to your app
@@ -52,9 +52,9 @@ namespace Usergrid.Notifications
//TODO: change me to your notifier name
notifier = "windows";
//TODO: change me to your user
- user = "mobileuser";
+ user = "test";
//TODO: change me to your password
- password = "P@ssw0rd1";
+ password = "test";
this.NavigationCacheMode = NavigationCacheMode.Required;
usergrid = new Client.Usergrid(serverUrl, org, app, user, password, notifier);
@@ -79,7 +79,7 @@ namespace Usergrid.Notifications
// If you are using the NavigationHelper provided by some templates,
// this event is handled for you.
}
-
+
private void pushText_TextChanged(object sender, TextChangedEventArgs e)
{
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6066b6e9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 625f4e7..d952d78 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -93,7 +93,6 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
if(polled!=null) {
drainList.add(polled);
producerQueue.drainTo(drainList, config.getIndexBufferSize());
- System.out.println("Consumer Thread" + Thread.currentThread().getName());
for(IndexOperationMessage drained : drainList){
subscriber.onNext(drained);
}
@@ -116,7 +115,6 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
.doOnNext(new Action1<List<IndexOperationMessage>>() {
@Override
public void call(List<IndexOperationMessage> containerList) {
- System.out.println("Buffered Consumer Thread" + Thread.currentThread().getName());
if (containerList.size() > 0) {
flushMeter.mark(containerList.size());
Timer.Context time = flushTimer.time();
[10/31] incubator-usergrid git commit: Added unique timers for each
method. Also added TODO for places where I was unsure if a graphite timer was
needed.
Posted by sf...@apache.org.
Added unique timers for each method. Also added TODO for places where I was unsure if a graphite timer was needed.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d5429098
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d5429098
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d5429098
Branch: refs/heads/USERGRID-405
Commit: d54290986f8eacf3e7415223ebd7ab239f7e9715
Parents: cb35127
Author: GERey <gr...@apigee.com>
Authored: Thu Mar 5 15:57:24 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Thu Mar 5 15:57:24 2015 -0800
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 83 ++++++++++++++------
.../corepersistence/CpRelationManager.java | 15 ++--
.../index/impl/EsEntityIndexImpl.java | 38 +++++----
3 files changed, 90 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5429098/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index e7efa6f..fa16f07 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -189,8 +189,6 @@ public class CpEntityManager implements EntityManager {
private boolean skipAggregateCounters;
- private Timer corePersistanceTimer;
-
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -206,7 +204,6 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
this.emf = ( CpEntityManagerFactory ) emf;
- corePersistanceTimer = this.emf.getMetricsFactory().getTimer( CpEntityManager.class, "cp.entity.manager.timer" );
this.managerCache = this.emf.getManagerCache();
this.applicationId = applicationId;
@@ -310,7 +307,7 @@ public class CpEntityManager implements EntityManager {
Entity entity = batchCreate( m, entityType, null, properties, importId, timestampUuid );
//Adding graphite metrics
- Timer.Context timeCassCreation = corePersistanceTimer.time();
+ Timer.Context timeCassCreation = this.emf.getMetricsFactory().getTimer( CpEntityManager.class, "cp.entity.create.timer" ).time();
m.execute();
timeCassCreation.stop();
@@ -344,7 +341,9 @@ public class CpEntityManager implements EntityManager {
A entity = batchCreate( m, entityType, entityClass, properties, importId, timestampUuid );
//Adding graphite metrics
- Timer.Context timeEntityCassCreation = corePersistanceTimer.time();
+ Timer.Context timeEntityCassCreation = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.create.batch.timer" ).time();
m.execute();
timeEntityCassCreation.stop();
@@ -987,6 +986,7 @@ public class CpEntityManager implements EntityManager {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion()
} );
+ //TODO: does this call and others like it need a graphite reporter?
cpEntity = ecm.write( cpEntity ).toBlockingObservable().last();
logger.debug( "Wrote {}:{} version {}", new Object[] {
@@ -994,7 +994,9 @@ public class CpEntityManager implements EntityManager {
} );
//Adding graphite metrics
- Timer.Context timeESBatch = corePersistanceTimer.time();
+ Timer.Context timeESBatch = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.es.delete.property.timer" ).time();
BetterFuture future = ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
timeESBatch.stop();
// update in all containing collections and connection indexes
@@ -1034,7 +1036,9 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, elementValue, false, timestampUuid );
//Adding graphite metrics
- Timer.Context timeDictionaryCreation = corePersistanceTimer.time();
+ Timer.Context timeDictionaryCreation = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.add.dictionary.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeDictionaryCreation.stop();
}
@@ -1058,7 +1062,9 @@ public class CpEntityManager implements EntityManager {
}
//Adding graphite metrics
- Timer.Context timeAddingSetDictionary = corePersistanceTimer.time();
+ Timer.Context timeAddingSetDictionary = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.add.dictionary.set.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeAddingSetDictionary.stop();
}
@@ -1083,7 +1089,9 @@ public class CpEntityManager implements EntityManager {
}
//Adding graphite metrics
- Timer.Context timeMapDictionary = corePersistanceTimer.time();
+ Timer.Context timeMapDictionary = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.add.dictionary.map.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeMapDictionary.stop();
}
@@ -1263,7 +1271,9 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, true, timestampUuid );
//Adding graphite metrics
- Timer.Context timeRemoveDictionary = corePersistanceTimer.time();
+ Timer.Context timeRemoveDictionary = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.remove.dictionary.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRemoveDictionary.stop();
@@ -1602,7 +1612,9 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), Schema.DICTIONARY_ROLENAMES, null,
timestamp );
//Adding graphite metrics
- Timer.Context timeCreateBatchRole= corePersistanceTimer.time();
+ Timer.Context timeCreateBatchRole= this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.create.role.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeCreateBatchRole.stop();
@@ -1619,7 +1631,9 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
- Timer.Context timeGrantRolePermission = corePersistanceTimer.time();
+ Timer.Context timeGrantRolePermission = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.create.role.permission.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGrantRolePermission.stop();
}
@@ -1637,7 +1651,9 @@ public class CpEntityManager implements EntityManager {
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp);
}
//Adding graphite metrics
- Timer.Context timeGrantRolePermissions = corePersistanceTimer.time();
+ Timer.Context timeGrantRolePermissions = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.create.role.permissions.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGrantRolePermissions.stop();
@@ -1671,12 +1687,14 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, timestamp );
//Adding graphite metrics
- Timer.Context timeRevokeRolePermission = corePersistanceTimer.time();
+ Timer.Context timeRevokeRolePermission = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.revoke.role.permissions.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRevokeRolePermission.stop();
}
-
+//TODO: does this need graphite monitoring
@Override
public Set<String> getRolePermissions( String roleName ) throws Exception {
roleName = roleName.toLowerCase();
@@ -1684,7 +1702,7 @@ public class CpEntityManager implements EntityManager {
ApplicationCF.ENTITY_DICTIONARIES, getRolePermissionsKey( roleName ) );
}
-
+//TODO: does this need graphite monitoring
@Override
public void deleteRole( String roleName ) throws Exception {
roleName = roleName.toLowerCase();
@@ -1739,7 +1757,9 @@ public class CpEntityManager implements EntityManager {
getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
- Timer.Context timeGroupRolePermission = corePersistanceTimer.time();
+ Timer.Context timeGroupRolePermission = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.grant.group.permission.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGroupRolePermission.stop();
}
@@ -1754,7 +1774,9 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( groupId, roleName ), permission, timestamp );
//Adding graphite metrics
- Timer.Context timeRevokeGroupRolePermission = corePersistanceTimer.time();
+ Timer.Context timeRevokeGroupRolePermission = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.revoke.group.permission.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRevokeGroupRolePermission.stop();
}
@@ -1906,7 +1928,10 @@ public class CpEntityManager implements EntityManager {
category, counterName, value, cassandraTimestamp / 1000, cassandraTimestamp );
//Adding graphite metrics
- Timer.Context timeIncrementAggregateCounters = corePersistanceTimer.time();
+ Timer.Context timeIncrementAggregateCounters = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.increment.aggregate.counters.timer" )
+ .time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
timeIncrementAggregateCounters.stop();
}
@@ -1934,7 +1959,9 @@ public class CpEntityManager implements EntityManager {
q.setRange( start, finish, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeGetAggregateCounters = corePersistanceTimer.time();
+ Timer.Context timeGetAggregateCounters = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.get.aggregate.counters.timer" ).time();
QueryResult<CounterSlice<Long>> r = q.setKey(
counterUtils.getAggregateCounterRow( counterName, userId, groupId, queueId, category, resolution ) )
.execute();
@@ -2009,7 +2036,9 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( APPLICATION_AGGREGATE_COUNTERS.toString() );
q.setRange( start, finish, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeGetAggregateCounters = corePersistanceTimer.time();
+ Timer.Context timeGetAggregateCounters = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.get.aggregate.counters.query.timer" ).time();
QueryResult<CounterRows<String, Long>> rows = q.setKeys( selections.keySet() ).execute();
timeGetAggregateCounters.stop();
@@ -2136,7 +2165,9 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( ENTITY_COUNTERS.toString() );
q.setRange( null, null, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeEntityCounters = corePersistanceTimer.time();
+ Timer.Context timeEntityCounters = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.get.entity.counters.timer" ).time();
QueryResult<CounterSlice<String>> r = q.setKey( entityId ).execute();
timeEntityCounters.stop();
for ( HCounterColumn<String> column : r.get().getColumns() ) {
@@ -2163,7 +2194,9 @@ public class CpEntityManager implements EntityManager {
m, applicationId, userId, groupId, null, category, counters, timestamp );
//Adding graphite metrics
- Timer.Context timeIncrementCounters = corePersistanceTimer.time();
+ Timer.Context timeIncrementCounters = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.increment.aggregate.counters.timer" ).time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
timeIncrementCounters.stop();
}
@@ -2897,7 +2930,9 @@ public class CpEntityManager implements EntityManager {
// batch.index(appAllTypesScope, memberEntity);
//Adding graphite metrics
- Timer.Context timeIndexEntityCollection = corePersistanceTimer.time();
+ Timer.Context timeIndexEntityCollection = this.emf.getMetricsFactory()
+ .getTimer( CpEntityManager.class,
+ "cp.entity.es.index.entity.to.collection.timer" ).time();
batch.execute();
timeIndexEntityCollection.stop();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5429098/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 499f841..f5d59d9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -121,7 +121,6 @@ import rx.functions.Func1;
import static java.util.Arrays.asList;
import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.corepersistence.util.CpEntityMapUtils.entityToCpEntity;
import static org.apache.usergrid.corepersistence.util.CpNamingUtils.getCollectionScopeNameFromEntityType;
import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
@@ -190,8 +189,6 @@ public class CpRelationManager implements RelationManager {
private ResultsLoaderFactory resultsLoaderFactory;
- private Timer cpRelationTimer;
-
private MetricsFactory metricsFactory;
public CpRelationManager() {}
@@ -224,7 +221,6 @@ public class CpRelationManager implements RelationManager {
this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
this.indexBucketLocator = indexBucketLocator; // TODO: this also
this.metricsFactory = metricsFactory;
- this.cpRelationTimer = metricsFactory.getTimer( CpRelationManager.class, "relation.manager.timer" );
// load the Core Persistence version of the head entity as well
this.headEntityScope = getCollectionScopeNameFromEntityType(
@@ -449,7 +445,8 @@ public class CpRelationManager implements RelationManager {
} ).count().toBlocking().lastOrDefault( 0 );
//Adding graphite metrics
- Timer.Context timeElasticIndexBatch = cpRelationTimer.time();
+ Timer.Context timeElasticIndexBatch = metricsFactory
+ .getTimer( CpRelationManager.class, "relation.manager.es.update.collection" ).time();
entityIndexBatch.execute();
timeElasticIndexBatch.stop();
@@ -1075,7 +1072,8 @@ public class CpRelationManager implements RelationManager {
Mutator<ByteBuffer> m = createMutator( ko, be );
batchUpdateEntityConnection( m, false, connection, UUIDGenerator.newTimeUUID() );
//Added Graphite Metrics
- Timer.Context timeElasticIndexBatch = cpRelationTimer.time();
+ Timer.Context timeElasticIndexBatch = metricsFactory
+ .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" ).time();
batchExecute( m, CassandraService.RETRY_COUNT );
timeElasticIndexBatch.stop();
@@ -1251,7 +1249,8 @@ public class CpRelationManager implements RelationManager {
m, true, ( ConnectionRefImpl ) connectionRef, UUIDGenerator.newTimeUUID() );
//Added Graphite Metrics
- Timer.Context timeDeleteConnections = cpRelationTimer.time();
+ Timer.Context timeDeleteConnections = metricsFactory
+ .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" ).time();
batchExecute( m, CassandraService.RETRY_COUNT );
timeDeleteConnections.stop();
@@ -1309,7 +1308,7 @@ public class CpRelationManager implements RelationManager {
// batch.deindex( allTypesIndexScope, targetEntity );
//Added Graphite Metrics
- Timer.Context timeDeleteConnection = cpRelationTimer.time();
+ Timer.Context timeDeleteConnection = metricsFactory.getTimer( CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" ).time();
batch.execute();
timeDeleteConnection.stop();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d5429098/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index f85692f..2cbe51f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -126,7 +126,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private EsIndexCache aliasCache;
- private final Timer indexTimer;
+// private final Timer indexTimer;
@Inject
@@ -139,12 +139,11 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, appScope);
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier( config, appScope );
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
this.metricsFactory = metricsFactory;
- this.indexTimer = metricsFactory.getTimer( EsEntityIndexImpl.class, "entity.index.timer" );
}
@Override
@@ -176,7 +175,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.build();
//Added For Graphite Metrics
- Timer.Context timeNewIndexCreation = indexTimer.time();
+ Timer.Context timeNewIndexCreation = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" ).time();
final CreateIndexResponse cir = admin.indices().prepareCreate(indexName)
.setSettings(settings)
.execute()
@@ -209,7 +209,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
for (String currentIndex : indexNames){
//Added For Graphite Metrics
- Timer.Context timeRemoveAlias = indexTimer.time();
+ Timer.Context timeRemoveAlias = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" ).time();
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
alias.getWriteAlias()).execute().actionGet().isAcknowledged();
timeRemoveAlias.stop();
@@ -217,7 +218,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
//Added For Graphite Metrics
- Timer.Context timeAddReadAlias = indexTimer.time();
+ Timer.Context timeAddReadAlias = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" ).time();
// add read alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
@@ -225,7 +227,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
logger.info("Created new read Alias Name [{}] ACK=[{}]", alias.getReadAlias(), isAck);
//Added For Graphite Metrics
- Timer.Context timeAddWriteAlias = indexTimer.time();
+ Timer.Context timeAddWriteAlias = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" ).time();
//add write alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
@@ -296,7 +299,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
XContentFactory.jsonBuilder(), "_default_");
//Added For Graphite Metrics
- Timer.Context timePutIndex = indexTimer.time();
+ Timer.Context timePutIndex = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" ).time();
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
// set mapping as the default for all types
@@ -389,7 +393,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
- Timer.Context timeSearch = indexTimer.time();
+ Timer.Context timeSearch = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" ).time();
searchResponse = srb.execute().actionGet();
timeSearch.stop();
}
@@ -417,7 +422,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
- Timer.Context timeSearchCursor = indexTimer.time();
+ Timer.Context timeSearchCursor = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" ).time();
searchResponse = ssrb.execute().actionGet();
timeSearchCursor.stop();
}
@@ -489,7 +495,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return true;
}
//Added For Graphite Metrics
- Timer.Context timeRefreshIndex = indexTimer.time();
+ Timer.Context timeRefreshIndex = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" ).time();
esProvider.getClient().admin().indices().prepareRefresh( indexes ).execute().actionGet();
timeRefreshIndex.stop();
logger.debug("Refreshed indexes: {}", StringUtils.join(indexes, ", "));
@@ -535,7 +542,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final SearchResponse searchResponse;
try {
//Added For Graphite Metrics
- Timer.Context timeEntityIndex = indexTimer.time();
+ Timer.Context timeEntityIndex = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" ).time();
searchResponse = srb.execute().actionGet();
timeEntityIndex.stop();
}
@@ -560,7 +568,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
//Added For Graphite Metrics
- final Timer.Context timeDeleteAllVersions = indexTimer.time();
+ final Timer.Context timeDeleteAllVersions = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" ).time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
@@ -601,7 +610,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//Added For Graphite Metrics
//Checks the time from the execute to the response below
- final Timer.Context timeDeletePreviousVersions = indexTimer.time();
+ final Timer.Context timeDeletePreviousVersions = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" ).time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
[03/31] incubator-usergrid git commit: Added timer to elasticsearch
index addition.
Posted by sf...@apache.org.
Added timer to elasticsearch index addition.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/00e7ca4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/00e7ca4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/00e7ca4b
Branch: refs/heads/USERGRID-405
Commit: 00e7ca4b599e14953e01dc608f90de7a4cdcbd9f
Parents: c1f5b78
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 4 15:03:15 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 4 15:03:15 2015 -0800
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexImpl.java | 16 +++++++++++++++-
1 file changed, 15 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/00e7ca4b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 49026e2..4200d13 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -18,6 +18,8 @@
package org.apache.usergrid.persistence.index.impl;
+import com.codahale.metrics.Meter;
+import com.codahale.metrics.Timer;
import com.google.common.collect.ImmutableMap;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
@@ -26,6 +28,7 @@ import com.google.inject.assistedinject.Assisted;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -121,9 +124,16 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private EsIndexCache aliasCache;
+ private final Timer flushTimer;
+
+ //private final Meter flushMeter;
+
+
@Inject
- public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config, final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider, final EsIndexCache indexCache) {
+ public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
+ final IndexBufferProducer indexBatchBufferProducer, final EsProvider provider,
+ final EsIndexCache indexCache, final MetricsFactory metricsFactory) {
this.indexBatchBufferProducer = indexBatchBufferProducer;
ValidationUtils.validateApplicationScope( appScope );
this.applicationScope = appScope;
@@ -134,6 +144,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
+ this.flushTimer = metricsFactory.getTimer( EsEntityIndexImpl.class, "entity.index.flush" );
}
@Override
@@ -163,10 +174,13 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.put("index.number_of_replicas", numberOfReplicas)
.put("action.write_consistency", writeConsistency )
.build();
+
+ Timer.Context timeNewIndexCreation = flushTimer.time();
final CreateIndexResponse cir = admin.indices().prepareCreate(indexName)
.setSettings(settings)
.execute()
.actionGet();
+ timeNewIndexCreation.stop();
logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
} catch (IndexAlreadyExistsException e) {
logger.info("Index Name [{}] already exists", indexName);
[18/31] incubator-usergrid git commit: Fixes lookup cache bug
Posted by sf...@apache.org.
Fixes lookup cache bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ecc1ddee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ecc1ddee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ecc1ddee
Branch: refs/heads/USERGRID-405
Commit: ecc1ddeed2f2e54a66c2b5a56eed689593380820
Parents: 4c8727c
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 16:10:33 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 16:10:33 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 31 ++++++++++----------
1 file changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ecc1ddee/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 16de53e..e0029c1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -241,10 +241,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
getSetup().setupApplicationKeyspace( applicationId, appName );
- final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( name );
+ final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
- final UUID orgUuid;
+ UUID orgUuid;
if ( !cachedValue.isPresent() ) {
@@ -252,22 +252,22 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// create new org because the specified one does not exist
final String orgName = organizationName;
- final Entity orgInfo;
+
try {
- orgInfo = em.create( "organization", new HashMap<String, Object>() {{
+ final Entity orgInfo = em.create( "organization", new HashMap<String, Object>() {{
put( PROPERTY_NAME, orgName );
}} );
+ orgUuid = orgInfo.getUuid();
+ //evit so it's re-loaded later
+ orgApplicationCache.evictOrgId( name );
}
catch ( DuplicateUniquePropertyExistsException e ) {
- throw new OrganizationAlreadyExistsException( orgName );
+ //swallow, if it exists, just get it
+ orgApplicationCache.evictOrgId( organizationName );
+ orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
}
- em.refreshIndex();
- orgUuid = orgInfo.getUuid();
-
- //evit so it's re-loaded later
- orgApplicationCache.evictOrgId( name );
} else{
orgUuid = cachedValue.get();
}
@@ -281,11 +281,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
put( "organizationUuid", orgId );
}};
- try{
- em.create( "appinfo", appInfoMap );
- }catch(DuplicateUniquePropertyExistsException e){
- throw new ApplicationAlreadyExistsException( appName );
- }
+ try {
+ em.create( "appinfo", appInfoMap );
+ }
+ catch ( DuplicateUniquePropertyExistsException e ) {
+ throw new ApplicationAlreadyExistsException( appName );
+ }
em.refreshIndex();
// create application entity
[28/31] incubator-usergrid git commit: Changes index declaration to
be explicit on the index
Posted by sf...@apache.org.
Changes index declaration to be explicit on the index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bcc52284
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bcc52284
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bcc52284
Branch: refs/heads/USERGRID-405
Commit: bcc52284f0c2322d8cd219e71adc16f0f0385cfc
Parents: 5decb0d
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 9 00:08:15 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 9 00:08:15 2015 -0600
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 22 ++++++++++----------
.../persistence/index/impl/IndexingUtils.java | 5 +----
2 files changed, 12 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bcc52284/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 80fede9..c9f5590 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -50,6 +50,7 @@ import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
@@ -93,6 +94,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private static final Logger logger = LoggerFactory.getLogger( EsEntityIndexImpl.class );
private static final AtomicBoolean mappingsCreated = new AtomicBoolean( false );
+ public static final String DEFAULT_TYPE = "_default_";
private final IndexIdentifier.IndexAlias alias;
private final IndexIdentifier indexIdentifier;
@@ -193,10 +195,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
String normalizedSuffix = StringUtils.isNotEmpty(indexSuffix) ? indexSuffix : null;
try {
- if (!mappingsCreated.getAndSet(true)) {
- createMappings();
- }
-
//get index name with suffix attached
String indexName = indexIdentifier.getIndex(normalizedSuffix);
@@ -217,6 +215,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.actionGet();
timeNewIndexCreation.stop();
+ //create the mappings
+ createMappings( indexName );
+
//ONLY add the alias if we create the index, otherwise we're going to overwrite production settings
/**
@@ -225,6 +226,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that
//may have been set via other administrative endpoint
+
addAlias(normalizedSuffix);
testNewIndex();
@@ -354,18 +356,16 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
* Setup ElasticSearch type mappings as a template that applies to all new indexes.
* Applies to all indexes that* start with our prefix.
*/
- private void createMappings() throws IOException {
+ private void createMappings(final String indexName) throws IOException {
XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_");
+ XContentFactory.jsonBuilder(), DEFAULT_TYPE );
+
//Added For Graphite Metrics
Timer.Context timePutIndex = mappingTimer.time();
- PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
- .preparePutTemplate("usergrid_template")
- // set mapping as the default for all types
- .setTemplate(config.getIndexPrefix() + "*").addMapping( "_default_", xcb )
- .execute().actionGet();
+ PutMappingResponse pitr = esProvider.getClient().admin().indices().preparePutMapping( indexName ).setType(
+ DEFAULT_TYPE ).setSource( xcb ).execute().actionGet();
timePutIndex.stop();
if ( !pitr.isAcknowledged() ) {
throw new IndexException( "Unable to create default mappings" );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bcc52284/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 14dca3e..be0c96c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -151,9 +151,8 @@ public class IndexingUtils {
.startObject()
- .startObject(type)
/** add routing "_routing":{ "required":false, "path":"ug_entityId" **/
- .startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
+ //.startObject("_routing").field("required",true).field("path",ENTITYID_ID_FIELDNAME).endObject()
.startArray("dynamic_templates")
// we need most specific mappings first since it's a stop on match algorithm
@@ -224,8 +223,6 @@ public class IndexingUtils {
.endArray()
- .endObject()
-
.endObject();
return builder;
[22/31] incubator-usergrid git commit: move timers up
Posted by sf...@apache.org.
move timers up
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5e0ce627
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5e0ce627
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5e0ce627
Branch: refs/heads/USERGRID-405
Commit: 5e0ce6276e978fb63eed6ac0b27521737169c6fd
Parents: 5cbfafe
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 17:46:40 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 17:46:40 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 83 +++++++++-----------
1 file changed, 38 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5e0ce627/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index fa16f07..f7a9f8b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -188,6 +188,9 @@ public class CpEntityManager implements EntityManager {
private CounterUtils counterUtils;
private boolean skipAggregateCounters;
+ private MetricsFactory metricsFactory;
+ private Timer aggCounterTimer;
+ private Timer entCreateTimer;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -211,7 +214,10 @@ public class CpEntityManager implements EntityManager {
this.cass = this.emf.getCassandraService();
this.counterUtils = this.emf.getCounterUtils();
-
+ this.metricsFactory = this.emf.getMetricsFactory();
+ this.aggCounterTimer =this.metricsFactory.getTimer( CpEntityManager.class,
+ "cp.entity.get.aggregate.counters.timer" );
+ this.entCreateTimer =this.metricsFactory.getTimer( CpEntityManager.class, "cp.entity.create.timer" );
// set to false for now
this.skipAggregateCounters = false;
@@ -307,7 +313,7 @@ public class CpEntityManager implements EntityManager {
Entity entity = batchCreate( m, entityType, null, properties, importId, timestampUuid );
//Adding graphite metrics
- Timer.Context timeCassCreation = this.emf.getMetricsFactory().getTimer( CpEntityManager.class, "cp.entity.create.timer" ).time();
+ Timer.Context timeCassCreation = entCreateTimer.time();
m.execute();
timeCassCreation.stop();
@@ -341,9 +347,8 @@ public class CpEntityManager implements EntityManager {
A entity = batchCreate( m, entityType, entityClass, properties, importId, timestampUuid );
//Adding graphite metrics
- Timer.Context timeEntityCassCreation = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.create.batch.timer" ).time();
+ Timer.Context timeEntityCassCreation = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.create.batch.timer").time();
m.execute();
timeEntityCassCreation.stop();
@@ -688,7 +693,7 @@ public class CpEntityManager implements EntityManager {
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
CpRelationManager rmi = new CpRelationManager();
- rmi.init( this, emf, applicationId, entityRef, null, emf.getMetricsFactory() );
+ rmi.init( this, emf, applicationId, entityRef, null, metricsFactory );
return rmi;
}
@@ -994,9 +999,8 @@ public class CpEntityManager implements EntityManager {
} );
//Adding graphite metrics
- Timer.Context timeESBatch = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.es.delete.property.timer" ).time();
+ Timer.Context timeESBatch = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.es.delete.property.timer").time();
BetterFuture future = ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
timeESBatch.stop();
// update in all containing collections and connection indexes
@@ -1036,9 +1040,8 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, elementValue, false, timestampUuid );
//Adding graphite metrics
- Timer.Context timeDictionaryCreation = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.add.dictionary.timer" ).time();
+ Timer.Context timeDictionaryCreation = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.add.dictionary.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeDictionaryCreation.stop();
}
@@ -1062,9 +1065,8 @@ public class CpEntityManager implements EntityManager {
}
//Adding graphite metrics
- Timer.Context timeAddingSetDictionary = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.add.dictionary.set.timer" ).time();
+ Timer.Context timeAddingSetDictionary = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.add.dictionary.set.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeAddingSetDictionary.stop();
}
@@ -1089,9 +1091,8 @@ public class CpEntityManager implements EntityManager {
}
//Adding graphite metrics
- Timer.Context timeMapDictionary = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.add.dictionary.map.timer" ).time();
+ Timer.Context timeMapDictionary = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.add.dictionary.map.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeMapDictionary.stop();
}
@@ -1271,9 +1272,8 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, true, timestampUuid );
//Adding graphite metrics
- Timer.Context timeRemoveDictionary = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.remove.dictionary.timer" ).time();
+ Timer.Context timeRemoveDictionary = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.remove.dictionary.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRemoveDictionary.stop();
@@ -1612,9 +1612,8 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), Schema.DICTIONARY_ROLENAMES, null,
timestamp );
//Adding graphite metrics
- Timer.Context timeCreateBatchRole= this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.create.role.timer" ).time();
+ Timer.Context timeCreateBatchRole= this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.create.role.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeCreateBatchRole.stop();
@@ -1631,9 +1630,8 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
- Timer.Context timeGrantRolePermission = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.create.role.permission.timer" ).time();
+ Timer.Context timeGrantRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.create.role.permission.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGrantRolePermission.stop();
}
@@ -1651,7 +1649,7 @@ public class CpEntityManager implements EntityManager {
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp);
}
//Adding graphite metrics
- Timer.Context timeGrantRolePermissions = this.emf.getMetricsFactory()
+ Timer.Context timeGrantRolePermissions = this.metricsFactory
.getTimer( CpEntityManager.class,
"cp.entity.create.role.permissions.timer" ).time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
@@ -1687,9 +1685,8 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, timestamp );
//Adding graphite metrics
- Timer.Context timeRevokeRolePermission = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.revoke.role.permissions.timer" ).time();
+ Timer.Context timeRevokeRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.revoke.role.permissions.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRevokeRolePermission.stop();
}
@@ -1757,9 +1754,8 @@ public class CpEntityManager implements EntityManager {
getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
//Adding graphite metrics
- Timer.Context timeGroupRolePermission = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.grant.group.permission.timer" ).time();
+ Timer.Context timeGroupRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.grant.group.permission.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeGroupRolePermission.stop();
}
@@ -1774,9 +1770,8 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( groupId, roleName ), permission, timestamp );
//Adding graphite metrics
- Timer.Context timeRevokeGroupRolePermission = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.revoke.group.permission.timer" ).time();
+ Timer.Context timeRevokeGroupRolePermission = this.metricsFactory.getTimer(CpEntityManager.class,
+ "cp.entity.revoke.group.permission.timer").time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
timeRevokeGroupRolePermission.stop();
}
@@ -1928,7 +1923,7 @@ public class CpEntityManager implements EntityManager {
category, counterName, value, cassandraTimestamp / 1000, cassandraTimestamp );
//Adding graphite metrics
- Timer.Context timeIncrementAggregateCounters = this.emf.getMetricsFactory()
+ Timer.Context timeIncrementAggregateCounters = this.metricsFactory
.getTimer( CpEntityManager.class,
"cp.entity.increment.aggregate.counters.timer" )
.time();
@@ -1959,9 +1954,7 @@ public class CpEntityManager implements EntityManager {
q.setRange( start, finish, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeGetAggregateCounters = this.emf.getMetricsFactory()
- .getTimer( CpEntityManager.class,
- "cp.entity.get.aggregate.counters.timer" ).time();
+ Timer.Context timeGetAggregateCounters = aggCounterTimer.time();
QueryResult<CounterSlice<Long>> r = q.setKey(
counterUtils.getAggregateCounterRow( counterName, userId, groupId, queueId, category, resolution ) )
.execute();
@@ -2036,7 +2029,7 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( APPLICATION_AGGREGATE_COUNTERS.toString() );
q.setRange( start, finish, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeGetAggregateCounters = this.emf.getMetricsFactory()
+ Timer.Context timeGetAggregateCounters = this.metricsFactory
.getTimer( CpEntityManager.class,
"cp.entity.get.aggregate.counters.query.timer" ).time();
QueryResult<CounterRows<String, Long>> rows = q.setKeys( selections.keySet() ).execute();
@@ -2165,7 +2158,7 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( ENTITY_COUNTERS.toString() );
q.setRange( null, null, false, ALL_COUNT );
//Adding graphite metrics
- Timer.Context timeEntityCounters = this.emf.getMetricsFactory()
+ Timer.Context timeEntityCounters = this.metricsFactory
.getTimer( CpEntityManager.class,
"cp.entity.get.entity.counters.timer" ).time();
QueryResult<CounterSlice<String>> r = q.setKey( entityId ).execute();
@@ -2194,7 +2187,7 @@ public class CpEntityManager implements EntityManager {
m, applicationId, userId, groupId, null, category, counters, timestamp );
//Adding graphite metrics
- Timer.Context timeIncrementCounters = this.emf.getMetricsFactory()
+ Timer.Context timeIncrementCounters = this.metricsFactory
.getTimer( CpEntityManager.class,
"cp.entity.increment.aggregate.counters.timer" ).time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
@@ -2930,7 +2923,7 @@ public class CpEntityManager implements EntityManager {
// batch.index(appAllTypesScope, memberEntity);
//Adding graphite metrics
- Timer.Context timeIndexEntityCollection = this.emf.getMetricsFactory()
+ Timer.Context timeIndexEntityCollection = this.metricsFactory
.getTimer( CpEntityManager.class,
"cp.entity.es.index.entity.to.collection.timer" ).time();
batch.execute();
[26/31] incubator-usergrid git commit: Moved add alias and test new
index into try so we don't keep moving the index alias
Posted by sf...@apache.org.
Moved add alias and test new index into try so we don't keep moving the index alias
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/fc07d9aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/fc07d9aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/fc07d9aa
Branch: refs/heads/USERGRID-405
Commit: fc07d9aa85e8c01cd0e3f122e0ac7375ec43a7a3
Parents: 3b409f6
Author: Todd Nine <tn...@apigee.com>
Authored: Sun Mar 8 20:14:37 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Sun Mar 8 21:07:58 2015 -0600
----------------------------------------------------------------------
.../persistence/index/impl/EsEntityIndexImpl.java | 16 ++++++++++++++--
1 file changed, 14 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/fc07d9aa/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d980401..80fede9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -216,14 +216,26 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.execute()
.actionGet();
timeNewIndexCreation.stop();
+
+ //ONLY add the alias if we create the index, otherwise we're going to overwrite production settings
+
+ /**
+ * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!!
+ */
+
+ //We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that
+ //may have been set via other administrative endpoint
+ addAlias(normalizedSuffix);
+
+ testNewIndex();
+
logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
} catch (IndexAlreadyExistsException e) {
logger.info("Index Name [{}] already exists", indexName);
}
- addAlias(normalizedSuffix);
- testNewIndex();
+
} catch (IndexAlreadyExistsException expected) {
// this is expected to happen if index already exists, it's a no-op and swallow
} catch (IOException e) {
[19/31] incubator-usergrid git commit: Adds put test to gatling
Posted by sf...@apache.org.
Adds put test to gatling
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b8f5bf64
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b8f5bf64
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b8f5bf64
Branch: refs/heads/USERGRID-405
Commit: b8f5bf6468f95fde810ebde8dc1e436db04cc2ce
Parents: ecc1dde
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 16:42:19 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 16:42:19 2015 -0700
----------------------------------------------------------------------
.../datagenerators/EntityDataGenerator.scala | 25 +++++++
.../datagenerators/FeederGenerator.scala | 18 ++++-
.../usergrid/scenarios/EntityScenarios.scala | 8 +--
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 ++++++++++++++++++++
5 files changed, 122 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8f5bf64/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
index daa54be..cb024d3 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/EntityDataGenerator.scala
@@ -85,4 +85,29 @@
return Map("entity" -> new JSONObject(entity).toString())
}
+
+ def generateCustomEntityJSONString(): String = {
+
+ var entity: Map[String, String] = Map(
+ // "name" -> "fdsa",
+ "address" -> Utils.generateRandomInt(10000, 1000000).toString,
+ "city" -> Utils.generateRandomInt(10000, 1000000).toString,
+ "state" -> Utils.generateRandomInt(10000, 1000000).toString,
+ "zip" -> Utils.generateRandomInt(10000, 1000000).toString,
+ "phone" -> Utils.generateRandomInt(10000, 1000000).toString,
+ "businessname" -> Utils.generateRandomInt(0, 1).toString,
+ "menu" -> Utils.generateRandomInt(1, 1000000).toString,
+ "specials" -> Utils.generateRandomInt(1, 1000000).toString,
+ "profile" -> "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.",
+ "description" -> "Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat.",
+ "directions" -> Utils.generateRandomInt(18, 65).toString,
+ "atmosphere" -> Utils.generateRandomInt(48, 84).toString,
+ "bar" -> Utils.generateRandomInt(120, 350).toString,
+ "tables" -> Utils.generateRandomInt(50, 100000).toString,
+ "outdoor" -> Utils.generateRandomInt(50, 100000).toString
+ )
+
+ return new JSONObject(entity).toString();
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8f5bf64/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
index 46dfdde..97be06a 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/datagenerators/FeederGenerator.scala
@@ -155,8 +155,22 @@
//val rod = "rod"
val userFeeder = Iterator.from(seed).map(i=>EntityDataGenerator.generateCustomEntity())
return userFeeder
- }
-
+ }
+
+
+
+ /**
+ * Generate users forever
+ * @param seed The seed
+ * @return
+ */
+ def generateCustomEntityPutInfinite(seed:Int): Iterator[Map[String, Any]] = {
+ //val rod = "rod"
+ val userFeeder = Iterator.from(seed).map(i=>Map("entityName" -> i.toString.concat(UUID.randomUUID().toString), "entity" -> EntityDataGenerator.generateCustomEntityJSONString()));
+ return userFeeder
+ }
+
+
def testFeeder(seed:Int): Iterator[Map[String, String]] = {
var entity: Map[String, String] = EntityDataGenerator.generateCustomEntity();
Map("entity" -> entity)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8f5bf64/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityScenarios.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityScenarios.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityScenarios.scala
index 70d66fa..c3af32d 100644
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityScenarios.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/scenarios/EntityScenarios.scala
@@ -40,9 +40,9 @@ object EntityScenarios {
val putEntity = exec(
http("Put custom entity")
- .put(Settings.baseAppUrl+"/${collectionType}/${entityName}")
- .body(StringBody("{\"address\":\""+Utils.generateRandomInt(1, Settings.numEntities)+"\",\"phone\":\""+Utils.generateRandomInt(1, Settings.numEntities)+"\"}}"))
- .headers(Headers.jsonAuthorized)
+ .put(Settings.baseAppUrl+"/"+ Settings.collectionType+"/${entityName}")
+ .body(StringBody("""${entity}"""))
+ .headers(Headers.jsonAnonymous)
.check(status.is(200))
)
@@ -61,7 +61,7 @@ object EntityScenarios {
.headers(Headers.jsonAnonymous)
.check(status.is(200))
)
-
+
val postEntityWithToken = exec(
http("Post custom entity")
.post(Settings.baseAppUrl+"/"+ Settings.collectionType)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8f5bf64/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
index 949f65a..2619cad 100755
--- a/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/settings/Settings.scala
@@ -44,8 +44,8 @@ object Settings {
val numEntities:Int = Integer.getInteger("numEntities", 5000).toInt
val numDevices:Int = Integer.getInteger("numDevices", 4000).toInt
-
- val collectionType:String = System.getProperty("collectionType")
+
+ val collectionType:String = System.getProperty("collectionType", "defaultthings")
val rampTime:Int = Integer.getInteger("rampTime", 0).toInt // in seconds
val throttle:Int = Integer.getInteger("throttle", 50).toInt // in seconds
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b8f5bf64/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/PutCustomEntitySimulation.scala
----------------------------------------------------------------------
diff --git a/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/PutCustomEntitySimulation.scala b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/PutCustomEntitySimulation.scala
new file mode 100644
index 0000000..2b129fc
--- /dev/null
+++ b/stack/loadtests/src/main/scala/org/apache/usergrid/simulations/PutCustomEntitySimulation.scala
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.simulations
+
+import io.gatling.core.Predef._
+import io.gatling.core.scenario.Simulation
+import org.apache.usergrid.datagenerators.FeederGenerator
+import org.apache.usergrid.scenarios.EntityScenarios
+import org.apache.usergrid.settings.Settings
+
+/**
+ * PostCustomEntitySimulation - creates lots of custom entities
+ *
+ * Run this way:
+ * mvn gatling:execute -DrampTime=10 -DmaxPossibleUsers=10 -Dduration=120 -Dorg=yourorgname -Dapp=sandbox -Dbaseurl=https://api.usergrid.com -DadminUser=yourusername -DadminPassword='yourpassword' -Dgatling.simulationClass=org.apache.usergrid.simulations.PostCustomEntitySimulation -DcollectionType=yourcollection
+ *
+ *
+ */
+class PutCustomEntitySimulation extends Simulation {
+
+ if(!Settings.skipSetup) {
+ println("Begin setup")
+ println("These aren't the droids you are looking for...")
+ //exec(TokenScenarios.getManagementToken)
+ println("End Setup")
+ }else{
+ println("Skipping Setup")
+ }
+
+ val numEntities:Int = Settings.numEntities
+ val collectionType = Settings.collectionType
+ println("collection type = " + collectionType)
+ val rampTime:Int = Settings.rampTime
+ val throttle:Int = Settings.throttle
+ val feeder = FeederGenerator.generateCustomEntityPutInfinite(0)
+ val httpConf = Settings.httpConf
+
+ val scnToRun = scenario("PUT custom entities")
+ .feed(feeder)
+ .exec(EntityScenarios.putEntity)
+
+ /*
+ val scnToRun = scenario("POST custom entities")
+ .feed(feeder)
+ .doIfOrElse(session => session("token").as[String].nonEmpty(session)) {
+ exec(EntityScenarios.postEntityWithToken)
+ } {
+ exec(EntityScenarios.postEntity)
+ }
+*/
+
+
+ setUp(scnToRun.inject(
+ rampUsers(Settings.maxPossibleUsers) over Settings.rampTime,
+ constantUsersPerSec(Settings.maxPossibleUsers) during Settings.duration
+ ).protocols(httpConf)).maxDuration(Settings.duration)
+
+}
[02/31] incubator-usergrid git commit: Merge branch 'two-dot-o' into
USERGRID-432
Posted by sf...@apache.org.
Merge branch 'two-dot-o' into USERGRID-432
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1c0f4f14
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1c0f4f14
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1c0f4f14
Branch: refs/heads/USERGRID-405
Commit: 1c0f4f14a5b8e5833ec96c6eadced75b70cf684b
Parents: 0c15e9b c1f5b78
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 4 10:53:39 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 4 10:53:39 2015 -0700
----------------------------------------------------------------------
stack/rest/catalina_base/bin/setenv.sh | 8 -
stack/rest/catalina_base/conf/server.xml | 144 -
stack/rest/catalina_base/conf/tomcat-users.xml | 36 -
stack/rest/catalina_base/conf/web.xml | 4614 ------------------
.../lib/usergrid-deployment.properties | 113 -
.../webapps/host-manager/META-INF/context.xml | 27 -
.../webapps/host-manager/WEB-INF/jsp/401.jsp | 70 -
.../webapps/host-manager/WEB-INF/jsp/403.jsp | 84 -
.../webapps/host-manager/WEB-INF/jsp/404.jsp | 61 -
.../webapps/host-manager/WEB-INF/web.xml | 144 -
.../webapps/host-manager/images/add.gif | Bin 1037 -> 0 bytes
.../webapps/host-manager/images/asf-logo.gif | Bin 7279 -> 0 bytes
.../webapps/host-manager/images/code.gif | Bin 394 -> 0 bytes
.../webapps/host-manager/images/design.gif | Bin 608 -> 0 bytes
.../webapps/host-manager/images/docs.gif | Bin 261 -> 0 bytes
.../webapps/host-manager/images/fix.gif | Bin 345 -> 0 bytes
.../webapps/host-manager/images/tomcat.gif | Bin 1934 -> 0 bytes
.../webapps/host-manager/images/update.gif | Bin 627 -> 0 bytes
.../webapps/host-manager/images/void.gif | Bin 43 -> 0 bytes
.../webapps/host-manager/index.jsp | 18 -
.../webapps/host-manager/manager.xml | 26 -
.../webapps/manager/META-INF/context.xml | 27 -
.../webapps/manager/WEB-INF/jsp/401.jsp | 79 -
.../webapps/manager/WEB-INF/jsp/403.jsp | 94 -
.../webapps/manager/WEB-INF/jsp/404.jsp | 62 -
.../manager/WEB-INF/jsp/sessionDetail.jsp | 197 -
.../manager/WEB-INF/jsp/sessionsList.jsp | 172 -
.../webapps/manager/WEB-INF/web.xml | 209 -
.../webapps/manager/images/add.gif | Bin 1037 -> 0 bytes
.../webapps/manager/images/asf-logo.gif | Bin 7279 -> 0 bytes
.../webapps/manager/images/code.gif | Bin 394 -> 0 bytes
.../webapps/manager/images/design.gif | Bin 608 -> 0 bytes
.../webapps/manager/images/docs.gif | Bin 261 -> 0 bytes
.../webapps/manager/images/fix.gif | Bin 345 -> 0 bytes
.../webapps/manager/images/tomcat.gif | Bin 2066 -> 0 bytes
.../webapps/manager/images/update.gif | Bin 627 -> 0 bytes
.../webapps/manager/images/void.gif | Bin 43 -> 0 bytes
.../catalina_base/webapps/manager/index.jsp | 18 -
.../catalina_base/webapps/manager/status.xsd | 84 -
.../catalina_base/webapps/manager/xform.xsl | 125 -
stack/rest/catalina_base/webapps/portal | 1 -
.../manager/org/apache/jsp/index_jsp.java | 83 -
.../usergrid/rest/test/PropertiesResource.java | 2 +-
.../apache/usergrid/rest/SystemResourceIT.java | 60 +
.../usergrid/rest/management/AdminUsersIT.java | 981 ++--
.../rest/test/resource2point0/ClientSetup.java | 31 +-
.../rest/test/resource2point0/RestClient.java | 18 +
.../resource2point0/TestPropertiesResource.java | 47 +
.../endpoints/DatabaseResource.java | 39 +
.../endpoints/SetupResource.java | 46 +
.../endpoints/SystemResource.java | 36 +
.../endpoints/mgmt/ConfirmResource.java | 46 +
.../endpoints/mgmt/FeedResource.java | 48 +
.../endpoints/mgmt/OrgResource.java | 12 +-
.../endpoints/mgmt/PasswordResource.java | 61 +
.../endpoints/mgmt/ReactivateResource.java | 46 +
.../endpoints/mgmt/ResetResource.java | 42 +
.../endpoints/mgmt/TokenResource.java | 4 +-
.../endpoints/mgmt/UserResource.java | 80 +
.../endpoints/mgmt/UsersResource.java | 13 +-
.../test/resource2point0/model/Credentials.java | 4 +-
.../resources/usergrid-custom-test.properties | 6 +
.../cassandra/ManagementServiceImpl.java | 7 +-
63 files changed, 1076 insertions(+), 7049 deletions(-)
----------------------------------------------------------------------
[06/31] incubator-usergrid git commit: Added metrics for the deindex
operation.
Posted by sf...@apache.org.
Added metrics for the deindex operation.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ff46465d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ff46465d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ff46465d
Branch: refs/heads/USERGRID-405
Commit: ff46465d2fee3907ee41f47ad83e5244f85e514d
Parents: 0f203b6
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 4 15:54:49 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 4 15:54:49 2015 -0800
----------------------------------------------------------------------
.../index/impl/EsEntityIndexBatchImpl.java | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ff46465d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index c2a3fdc..d987b29 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.*;
import org.apache.usergrid.persistence.core.future.BetterFuture;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.*;
import org.elasticsearch.action.delete.DeleteRequestBuilder;
import org.elasticsearch.action.index.IndexRequestBuilder;
@@ -49,6 +50,9 @@ import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import com.codahale.metrics.*;
+import com.codahale.metrics.Timer;
+
import rx.Observable;
import rx.functions.Func1;
@@ -81,9 +85,12 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private final AliasedEntityIndex entityIndex;
private IndexOperationMessage container;
+ private final Timer batchTimer;
+
- public EsEntityIndexBatchImpl(final ApplicationScope applicationScope, final Client client,final IndexBufferProducer indexBatchBufferProducer,
- final IndexFig config, final AliasedEntityIndex entityIndex ) {
+ public EsEntityIndexBatchImpl(final ApplicationScope applicationScope, final Client client,
+ final IndexBufferProducer indexBatchBufferProducer,final IndexFig config,
+ final AliasedEntityIndex entityIndex,final MetricsFactory metricsFactory ) {
this.applicationScope = applicationScope;
this.client = client;
@@ -92,6 +99,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, applicationScope);
this.alias = indexIdentifier.getAlias();
this.refresh = config.isForcedRefresh();
+ this.batchTimer = metricsFactory.getTimer( EsEntityIndexBatchImpl.class, "entity.index.batch.timer" );
//constrained
this.container = new IndexOperationMessage();
}
@@ -167,6 +175,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
indexes = new String[]{indexIdentifier.getIndex(null)};
}
//get all indexes then flush everyone
+ Timer.Context timeDeindex = batchTimer.time();
Observable.from(indexes)
.map(new Func1<String, Object>() {
@Override
@@ -181,7 +190,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
return index;
}
}).toBlocking().last();
-
+ timeDeindex.stop();
log.debug("Deindexed Entity with index id " + indexId);
return this;
[12/31] incubator-usergrid git commit: windows push raw notifications
Posted by sf...@apache.org.
windows push raw notifications
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/44bbd6b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/44bbd6b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/44bbd6b9
Branch: refs/heads/USERGRID-405
Commit: 44bbd6b97546aeb9e29699cabab8e6f384f79914
Parents: 1c0f4f1
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 14:16:10 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 14:16:10 2015 -0700
----------------------------------------------------------------------
.../services/notifications/wns/WNSAdapter.java | 39 ++++++++++++++++++++
1 file changed, 39 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/44bbd6b9/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
index 9030fdd..34ae63f 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/wns/WNSAdapter.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.services.notifications.wns;
import ar.com.fernandospr.wns.WnsService;
import ar.com.fernandospr.wns.model.WnsBadge;
+import ar.com.fernandospr.wns.model.WnsRaw;
import ar.com.fernandospr.wns.model.WnsToast;
import ar.com.fernandospr.wns.model.builders.WnsBadgeBuilder;
import ar.com.fernandospr.wns.model.builders.WnsToastBuilder;
@@ -36,6 +37,7 @@ import org.apache.usergrid.services.notifications.TaskTracker;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.*;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -87,6 +89,11 @@ public class WNSAdapter implements ProviderAdapter {
}
service.pushBadge(providerId, badge);
break;
+ case "raw" :
+ WnsRaw raw = new WnsRaw();
+ raw.stream = toBytes( translatedNotification.getMessage() ) ;
+ service.pushRaw(providerId, raw);
+ break;
default : throw new IllegalArgumentException(translatedNotification.getType()+" does not match a valid notification type (toast,badge).");
}
}
@@ -97,6 +104,35 @@ public class WNSAdapter implements ProviderAdapter {
}
}
+ private byte[] toBytes(Object message) {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ObjectOutput out = null;
+ try {
+ if(message instanceof Serializable) {
+ out = new ObjectOutputStream(bos);
+ out.writeObject(message);
+ byte[] yourBytes = bos.toByteArray();
+ return yourBytes;
+ }else{
+ throw new RuntimeException("message is not serializable");
+ }
+ }catch (IOException e){
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ if (out != null) {
+ out.close();
+ }
+ } catch (IOException ex) {
+ // ignore close exception
+ }
+ try {
+ bos.close();
+ } catch (IOException ex) {
+ // ignore close exception
+ }
+ }
+ }
@Override
public void doneSendingNotifications() throws Exception {
@@ -120,6 +156,9 @@ public class WNSAdapter implements ProviderAdapter {
if (map.containsKey("badge")) {
translatedNotifications.add(new TranslatedNotification(map.get("badge"), "badge"));
}
+ if (map.containsKey("raw")) {
+ translatedNotifications.add(new TranslatedNotification(map.get("raw"), "raw"));
+ }
} else {
//{payloads:{winphone:"mymessage"}}
[13/31] incubator-usergrid git commit: windows raw notifications
client side
Posted by sf...@apache.org.
windows raw notifications client side
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/615a5afd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/615a5afd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/615a5afd
Branch: refs/heads/USERGRID-405
Commit: 615a5afdf24e7ef45196958783a775df80c3c003
Parents: 44bbd6b
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 14:20:00 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 14:20:00 2015 -0700
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 +++++++++++++++++++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 25 ++++++++++++++++----
4 files changed, 46 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615a5afd/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
index 3f4fb82..5efbb9b 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/IUsergridClient.cs
@@ -108,7 +108,7 @@ namespace Usergrid.Notifications.Client
/// <param name="message"></param>
/// <returns></returns>
Task<bool> SendToast(string message);
-
+ Task<bool> SendRaw(string message);
/// <summary>
/// Send a badge update
/// </summary>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615a5afd/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
index fb8860c..343e28b 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/Client/PushClient.cs
@@ -85,9 +85,28 @@ namespace Usergrid.Notifications.Client
return jsonResponse.StatusIsOk;
}
+
+ public async Task<bool> SendRaw(String message)
+ {
+ if (DeviceId == null)
+ {
+ throw new Exception("Please call PushClient.RegisterDevice first.");
+ }
+ var jsonObject = new JObject();
+ var payloads = new JObject();
+ var payload = new JObject();
+ payload.Add("raw", new JValue(message));
+ payloads.Add(Notifier, payload);
+ jsonObject.Add("payloads", payloads);
+ jsonObject.Add("debug", true);
+ var jsonResponse = await usergrid.SendAsync(HttpMethod.Post, String.Format("users/{1}/devices/{0}/notifications", this.DeviceId, userId), jsonObject);
+ return jsonResponse.StatusIsOk;
+ }
+
private async Task init()
{
channel = await PushNotificationChannelManager.CreatePushNotificationChannelForApplicationAsync().AsTask<PushNotificationChannel>();
+ channel.PushNotificationReceived += channel_PushNotificationReceived;
if (settings.Values[DEVICE_KEY] == null)
{
Guid uuid = await registerDevice(true);
@@ -113,6 +132,11 @@ namespace Usergrid.Notifications.Client
}
}
+ void channel_PushNotificationReceived(PushNotificationChannel sender, PushNotificationReceivedEventArgs args)
+ {
+ throw new NotImplementedException();
+ }
+
private async Task<JToken> GetDevice(Guid deviceId)
{
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615a5afd/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml
index f22c8b6..004eba9 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml
@@ -33,6 +33,7 @@
<Button x:Name="SendToast" Content="Toast" HorizontalAlignment="Left" Margin="114,88,0,0" VerticalAlignment="Top" Click="Button_Click" Width="39"/>
<Button x:Name="Badge" Content="Badge" HorizontalAlignment="Left" Margin="114,157,0,0" VerticalAlignment="Top" Click="Badge_Click"/>
<TextBlock x:Name="Result" HorizontalAlignment="Left" Margin="46,295,0,0" TextWrapping="Wrap" Text="Did it work? " VerticalAlignment="Top" Height="49" Width="261" FontSize="25" SelectionChanged="Result_SelectionChanged"/>
+ <Button Content="Raw" HorizontalAlignment="Left" Margin="114,223,0,0" VerticalAlignment="Top" Click="Button_Click_1"/>
</Grid>
</Page>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/615a5afd/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
----------------------------------------------------------------------
diff --git a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
index a550954..ccef239 100644
--- a/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
+++ b/sdks/dotnet/samples/notifications/Usergrid.Notifications/MainPage.xaml.cs
@@ -44,17 +44,17 @@ namespace Usergrid.Notifications
{
this.InitializeComponent();
//TODO: change me to your server
- serverUrl = "https://usergrid-push.example.com";
+ serverUrl = "http://10.0.1.20:8080/";
//TODO: change me to your org
- org = "ugvalidate";
+ org = "test-organization";
//TODO: change me to your app
- app = "sandbox";
+ app = "test-app";
//TODO: change me to your notifier name
- notifier = "winphone";
+ notifier = "windows";
//TODO: change me to your user
user = "mobileuser";
//TODO: change me to your password
- password = "******";
+ password = "P@ssw0rd1";
this.NavigationCacheMode = NavigationCacheMode.Required;
usergrid = new Client.Usergrid(serverUrl, org, app, user, password, notifier);
@@ -120,5 +120,20 @@ namespace Usergrid.Notifications
}
public AggregateException LastException { get; set; }
+
+ private async void Button_Click_1(object sender, RoutedEventArgs e)
+ {
+ Result.Text = "Sending....";
+
+ var message = this.pushText.Text;
+ if (await usergrid.Push.SendRaw(message))
+ {
+ Result.Text = "It did! :)";
+ }
+ else
+ {
+ Result.Text = "It did not! :(";
+ }
+ }
}
}
[11/31] incubator-usergrid git commit: Updated tomcat threads to be
configurable per core in cloud formation template
Posted by sf...@apache.org.
Updated tomcat threads to be configurable per core in cloud formation template
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/10ec6354
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/10ec6354
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/10ec6354
Branch: refs/heads/USERGRID-405
Commit: 10ec6354c9d6128b28811ba969d2ba3eec24b624
Parents: 710a1a3
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 13:35:17 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 13:35:17 2015 -0700
----------------------------------------------------------------------
.../src/main/dist/init_instance/init_rest_server.sh | 3 ---
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 6 +++---
stack/awscluster/ugcluster-cf.json | 8 ++++++++
3 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ec6354/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 3860cd4..cad4e74 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -68,9 +68,6 @@ export ACCEPT_COUNT=100
export NR_OPEN=1048576
export FILE_MAX=761773
-#Number of threads to allow per core
-export NUM_THREAD_PROC=25
-
#Get the number of processors
export NUM_PROC=$(nproc)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ec6354/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 619c0b4..1f7140c 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -50,9 +50,9 @@ def esShards = numEsNodes;
//This gives us 3 copies, which means we'll have a quorum with primary + 1 replica
def esReplicas = 1;
-def cassThreads = System.getenv().get("TOMCAT_THREADS")
+def tomcatThreads = System.getenv().get("TOMCAT_THREADS")
//temporarily set to equal since we now have a sane tomcat thread calculation
-def hystrixThreads = cassThreads
+def hystrixThreads = tomcatThreads
//if we end in -1, we remove it
def ec2Region = System.getenv().get("EC2_REGION")
@@ -99,7 +99,7 @@ cassandra.keyspace.strategy=org.apache.cassandra.locator.NetworkTopologyStrategy
cassandra.keyspace.replication=${cassEc2Region}:${replFactor}
cassandra.timeout=5000
-cassandra.connections=${cassThreads}
+cassandra.connections=${tomcatThreads}
hystrix.threadpool.graph_user.coreSize=${hystrixThreads}
hystrix.threadpool.graph_async.coreSize=${hystrixThreads}
usergrid.read.cl=${readConsistencyLevel}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ec6354/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 9bce545..7926929 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -46,6 +46,12 @@
],
"ConstraintDescription": "must be valid instance type."
},
+ "TomcatThreadsPerCore": {
+ "Description": "Number of threads to configure tomcat for per core",
+ "Type": "Number",
+ "Default": "50",
+ "MinValue": "1"
+ },
"KeyPair": {
"Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
"Type": "AWS::EC2::KeyPair::KeyName",
@@ -1171,6 +1177,8 @@
"\n",
"export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
"\n",
+ "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
+ "\n",
"export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
"export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" }, "\n",
"\n",
[31/31] incubator-usergrid git commit: merge from 416
Posted by sf...@apache.org.
merge from 416
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b6351134
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b6351134
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b6351134
Branch: refs/heads/USERGRID-405
Commit: b6351134e6d9eb0e1def02fe319aeb3b81db29ad
Parents: 0c3b439 673c057
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 9 09:25:03 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 9 09:25:03 2015 -0600
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 +++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 ++-
.../main/dist/init_instance/init_rest_server.sh | 3 -
.../src/main/groovy/configure_usergrid.groovy | 6 +-
stack/awscluster/ugcluster-cf.json | 8 +
.../corepersistence/CpEntityManager.java | 109 +++++++++-
.../corepersistence/CpEntityManagerFactory.java | 151 ++++++--------
.../corepersistence/CpRelationManager.java | 39 +++-
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++
.../corepersistence/results/EntityVerifier.java | 4 +-
.../results/FilteringLoader.java | 2 +-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
.../main/resources/usergrid-core-context.xml | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../impl/migration/GraphMigrationPlugin.java | 1 -
.../persistence/index/IndexBufferProducer.java | 10 +-
.../usergrid/persistence/index/IndexFig.java | 12 +-
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 9 +-
.../index/impl/EsEntityIndexImpl.java | 199 +++++++++++++++----
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++-
.../persistence/index/impl/IndexingUtils.java | 7 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../datagenerators/EntityDataGenerator.scala | 25 +++
.../datagenerators/FeederGenerator.scala | 18 +-
.../usergrid/scenarios/EntityScenarios.scala | 8 +-
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 +++++++
.../org/apache/usergrid/rest/IndexResource.java | 19 ++
.../services/notifications/wns/WNSAdapter.java | 39 ++++
.../resources/usergrid-services-context.xml | 8 +-
35 files changed, 947 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3b39bc9,12504ed..a897ad2
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -41,9 -46,8 +49,10 @@@ import org.apache.usergrid.persistence.
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+ import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
+import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
import org.apache.usergrid.persistence.core.util.Health;
@@@ -106,9 -97,10 +111,12 @@@ public class CpEntityManagerFactory imp
}
});
+ private final OrgApplicationCache orgApplicationCache;
+
+
private ManagerCache managerCache;
+
+
private DataMigrationManager dataMigrationManager;
private CassandraService cassandraService;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
index 22f34ff,0000000..ed51c27
mode 100644,000000..100644
--- a/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
+++ b/stack/corepersistence/graph/src/main/java/org/apache/usergrid/persistence/graph/serialization/impl/migration/GraphMigrationPlugin.java
@@@ -1,70 -1,0 +1,69 @@@
+/*
+ *
+ * *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ * *
+ *
+ */
+
+package org.apache.usergrid.persistence.graph.serialization.impl.migration;
+
+
+import java.util.Set;
+
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.migration.data.AbstractMigrationPlugin;
+import org.apache.usergrid.persistence.core.migration.data.DataMigration;
+import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
+import org.apache.usergrid.persistence.core.migration.data.PluginPhase;
+
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
- import sun.plugin2.main.server.Plugin;
+
+
+/**
+ * Migration plugin for the collection module
+ */
+@Singleton
+public class GraphMigrationPlugin extends AbstractMigrationPlugin<GraphNode> {
+
+ public static final String PLUGIN_NAME = "graph-data";
+
+
+
+ @Inject
+ public GraphMigrationPlugin(@GraphMigration
+ final Set<DataMigration<GraphNode>> entityDataMigrations,
+ final MigrationDataProvider<GraphNode> entityIdScopeDataMigrationProvider,
+ final MigrationInfoSerialization migrationInfoSerialization ) {
+ super( entityDataMigrations, entityIdScopeDataMigrationProvider, migrationInfoSerialization );
+ }
+
+
+ @Override
+ public String getName() {
+ return PLUGIN_NAME;
+ }
+
+
+ @Override
+ public PluginPhase getPhase() {
+ return PluginPhase.MIGRATE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index b1794f8,c9f5590..caa2fc7
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -44,11 -48,9 +48,12 @@@ import org.elasticsearch.action.admin.c
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksRequest;
import org.elasticsearch.action.admin.cluster.tasks.PendingClusterTasksResponse;
+import org.elasticsearch.action.admin.indices.alias.IndicesAliasesResponse;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesRequest;
+import org.elasticsearch.action.admin.indices.alias.get.GetAliasesResponse;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
+ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b6351134/stack/rest/src/main/java/org/apache/usergrid/rest/IndexResource.java
----------------------------------------------------------------------
[25/31] incubator-usergrid git commit: Fixes merge error
Posted by sf...@apache.org.
Fixes merge error
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3b409f67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3b409f67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3b409f67
Branch: refs/heads/USERGRID-405
Commit: 3b409f67f712ccee2ef28813a4e1af9cb0beb685
Parents: 046ba4e
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 18:05:25 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 18:05:25 2015 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexFactoryImpl.java | 9 +++--
.../index/impl/EsEntityIndexImpl.java | 35 +++++++++++---------
2 files changed, 27 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3b409f67/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 4bf8b59..3a9f790 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -24,6 +24,8 @@ import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
@@ -41,21 +43,24 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final EsProvider provider;
private final EsIndexCache indexCache;
private final IndexBufferProducer indexBatchBufferProducer;
+ private final MetricsFactory metricsFactory;
private LoadingCache<ApplicationScope, EntityIndex> eiCache =
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, EntityIndex>() {
public EntityIndex load( ApplicationScope scope ) {
- return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache);
+ return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache, metricsFactory);
}
} );
@Inject
public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
- final IndexBufferProducer indexBatchBufferProducer ){
+ final IndexBufferProducer indexBatchBufferProducer,
+ final MetricsFactory metricsFactory ){
this.config = config;
this.provider = provider;
this.indexCache = indexCache;
this.indexBatchBufferProducer = indexBatchBufferProducer;
+ this.metricsFactory = metricsFactory;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3b409f67/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 951f96b..d980401 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -241,25 +241,30 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
String[] indexNames = getIndexes(AliasType.Write);
- for (String currentIndex : indexNames){
+ for ( String currentIndex : indexNames ) {
final Timer.Context timeRemoveAlias = removeAliasTimer.time();
+
try {
- //Added For Graphite Metrics
+ //Added For Graphite Metrics
- isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
- alias.getWriteAlias()).execute().actionGet().isAcknowledged();
-
- logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck);
- } catch (AliasesMissingException aie) {
- logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, aie.getMessage());
- continue;
- }catch(InvalidAliasNameException iane) {
- logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, iane.getMessage());
- continue;
- }finally{
- timeRemoveAlias.stop();
+ isAck = adminClient.indices().prepareAliases().removeAlias( currentIndex, alias.getWriteAlias() )
+ .execute().actionGet().isAcknowledged();
+ logger.info( "Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck );
+ }
+ catch ( AliasesMissingException aie ) {
+ logger.info( "Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias,
+ aie.getMessage() );
+ continue;
+ }
+ catch ( InvalidAliasNameException iane ) {
+ logger.info( "Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias,
+ iane.getMessage() );
+ continue;
+ }
+ finally {
+ timeRemoveAlias.stop();
}
}
@@ -617,7 +622,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public void onResponse( DeleteByQueryResponse response) {
timeDeleteAllVersions.stop();
logger
- .debug("Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
+ .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
entityId.getUuid(), response.status().toString() );
checkDeleteByQueryResponse(tqb, response);
[08/31] incubator-usergrid git commit: Added timer for
cpEntityManager and CpRelationManager to cover commands made by entity
managers.
Posted by sf...@apache.org.
Added timer for cpEntityManager and CpRelationManager to cover commands made by entity managers.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d3b469ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d3b469ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d3b469ea
Branch: refs/heads/USERGRID-405
Commit: d3b469eae7047c1cca20ed7c18727ba78f2618b6
Parents: b3d4eb2
Author: GERey <gr...@apigee.com>
Authored: Thu Mar 5 11:28:05 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Thu Mar 5 11:28:05 2015 -0800
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 62 +++++++++++++++++++-
.../corepersistence/CpRelationManager.java | 25 +++++++-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
3 files changed, 82 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3b469ea/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 10e7926..4355638 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -309,6 +309,7 @@ public class CpEntityManager implements EntityManager {
Entity entity = batchCreate( m, entityType, null, properties, importId, timestampUuid );
+ //Adding graphite metrics
Timer.Context timeCassCreation = corePersistanceTimer.time();
m.execute();
timeCassCreation.stop();
@@ -342,7 +343,10 @@ public class CpEntityManager implements EntityManager {
A entity = batchCreate( m, entityType, entityClass, properties, importId, timestampUuid );
+ //Adding graphite metrics
+ Timer.Context timeEntityCassCreation = corePersistanceTimer.time();
m.execute();
+ timeEntityCassCreation.stop();
return entity;
}
@@ -989,8 +993,10 @@ public class CpEntityManager implements EntityManager {
cpEntity.getId().getType(), cpEntity.getId().getUuid(), cpEntity.getVersion()
} );
-
+ //Adding graphite metrics
+ Timer.Context timeESBatch = corePersistanceTimer.time();
BetterFuture future = ei.createBatch().index( defaultIndexScope, cpEntity ).execute();
+ timeESBatch.stop();
// update in all containing collections and connection indexes
CpRelationManager rm = ( CpRelationManager ) getRelationManager( entityRef );
rm.updateContainingCollectionAndCollectionIndexes( cpEntity );
@@ -1027,7 +1033,10 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, elementValue, false, timestampUuid );
+ //Adding graphite metrics
+ Timer.Context timeDictionaryCreation = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeDictionaryCreation.stop();
}
@@ -1048,7 +1057,10 @@ public class CpEntityManager implements EntityManager {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue, null, false, timestampUuid );
}
+ //Adding graphite metrics
+ Timer.Context timeAddingSetDictionary = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeAddingSetDictionary.stop();
}
@@ -1070,7 +1082,10 @@ public class CpEntityManager implements EntityManager {
elementValue.getValue(), false, timestampUuid );
}
+ //Adding graphite metrics
+ Timer.Context timeMapDictionary = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeMapDictionary.stop();
}
@@ -1247,8 +1262,11 @@ public class CpEntityManager implements EntityManager {
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementName, true, timestampUuid );
-
+ //Adding graphite metrics
+ Timer.Context timeRemoveDictionary = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeRemoveDictionary.stop();
+
}
@@ -1583,8 +1601,10 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addInsertToMutator( batch, ENTITY_DICTIONARIES,
CassandraPersistenceUtils.key( ownerId, DICTIONARY_SETS ), Schema.DICTIONARY_ROLENAMES, null,
timestamp );
-
+ //Adding graphite metrics
+ Timer.Context timeCreateBatchRole= corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeCreateBatchRole.stop();
return get( id, Role.class );
}
@@ -1598,7 +1618,10 @@ public class CpEntityManager implements EntityManager {
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
+ //Adding graphite metrics
+ Timer.Context timeGrantRolePermission = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeGrantRolePermission.stop();
}
@@ -1613,7 +1636,11 @@ public class CpEntityManager implements EntityManager {
CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, ByteBuffer.allocate( 0 ), timestamp);
}
+ //Adding graphite metrics
+ Timer.Context timeGrantRolePermissions = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeGrantRolePermissions.stop();
+
}
@@ -1643,7 +1670,10 @@ public class CpEntityManager implements EntityManager {
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be);
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( roleName ), permission, timestamp );
+ //Adding graphite metrics
+ Timer.Context timeRevokeRolePermission = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeRevokeRolePermission.stop();
}
@@ -1707,7 +1737,11 @@ public class CpEntityManager implements EntityManager {
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
+
+ //Adding graphite metrics
+ Timer.Context timeGroupRolePermission = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeGroupRolePermission.stop();
}
@@ -1719,7 +1753,10 @@ public class CpEntityManager implements EntityManager {
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
CassandraPersistenceUtils.addDeleteToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
getRolePermissionsKey( groupId, roleName ), permission, timestamp );
+ //Adding graphite metrics
+ Timer.Context timeRevokeGroupRolePermission = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( batch, CassandraService.RETRY_COUNT );
+ timeRevokeGroupRolePermission.stop();
}
@@ -1868,7 +1905,10 @@ public class CpEntityManager implements EntityManager {
counterUtils.batchIncrementAggregateCounters( m, applicationId, userId, groupId, null,
category, counterName, value, cassandraTimestamp / 1000, cassandraTimestamp );
+ //Adding graphite metrics
+ Timer.Context timeIncrementAggregateCounters = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
+ timeIncrementAggregateCounters.stop();
}
}
@@ -1893,9 +1933,12 @@ public class CpEntityManager implements EntityManager {
q.setColumnFamily( APPLICATION_AGGREGATE_COUNTERS.toString() );
q.setRange( start, finish, false, ALL_COUNT );
+ //Adding graphite metrics
+ Timer.Context timeGetAggregateCounters = corePersistanceTimer.time();
QueryResult<CounterSlice<Long>> r = q.setKey(
counterUtils.getAggregateCounterRow( counterName, userId, groupId, queueId, category, resolution ) )
.execute();
+ timeGetAggregateCounters.stop();
List<AggregateCounter> counters = new ArrayList<AggregateCounter>();
for ( HCounterColumn<Long> column : r.get().getColumns() ) {
@@ -1965,7 +2008,11 @@ public class CpEntityManager implements EntityManager {
MultigetSliceCounterQuery<String, Long> q = HFactory.createMultigetSliceCounterQuery( ko, se, le );
q.setColumnFamily( APPLICATION_AGGREGATE_COUNTERS.toString() );
q.setRange( start, finish, false, ALL_COUNT );
+ //Adding graphite metrics
+ Timer.Context timeGetAggregateCounters = corePersistanceTimer.time();
QueryResult<CounterRows<String, Long>> rows = q.setKeys( selections.keySet() ).execute();
+ timeGetAggregateCounters.stop();
+
List<AggregateCounterSet> countSets = new ArrayList<AggregateCounterSet>();
for ( CounterRow<String, Long> r : rows.get() ) {
@@ -2088,7 +2135,10 @@ public class CpEntityManager implements EntityManager {
SliceCounterQuery<UUID, String> q = createCounterSliceQuery( ko, ue, se );
q.setColumnFamily( ENTITY_COUNTERS.toString() );
q.setRange( null, null, false, ALL_COUNT );
+ //Adding graphite metrics
+ Timer.Context timeEntityCounters = corePersistanceTimer.time();
QueryResult<CounterSlice<String>> r = q.setKey( entityId ).execute();
+ timeEntityCounters.stop();
for ( HCounterColumn<String> column : r.get().getColumns() ) {
counters.put( column.getName(), column.getValue() );
}
@@ -2112,7 +2162,10 @@ public class CpEntityManager implements EntityManager {
counterUtils.batchIncrementAggregateCounters(
m, applicationId, userId, groupId, null, category, counters, timestamp );
+ //Adding graphite metrics
+ Timer.Context timeIncrementCounters = corePersistanceTimer.time();
CassandraPersistenceUtils.batchExecute( m, CassandraService.RETRY_COUNT );
+ timeIncrementCounters.stop();
}
}
@@ -2843,7 +2896,10 @@ public class CpEntityManager implements EntityManager {
//
// batch.index(appAllTypesScope, memberEntity);
+ //Adding graphite metrics
+ Timer.Context timeIndexEntityCollection = corePersistanceTimer.time();
batch.execute();
+ timeIndexEntityCollection.stop();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3b469ea/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 302f18d..786b0f5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -61,6 +61,7 @@ import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
import org.apache.usergrid.persistence.cassandra.index.NoOpIndexScanner;
import org.apache.usergrid.persistence.collection.CollectionScope;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.entities.Group;
import org.apache.usergrid.persistence.entities.User;
@@ -105,6 +106,7 @@ import org.apache.usergrid.utils.IndexUtils;
import org.apache.usergrid.utils.MapUtils;
import org.apache.usergrid.utils.UUIDUtils;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.yammer.metrics.annotation.Metered;
@@ -188,7 +190,7 @@ public class CpRelationManager implements RelationManager {
private ResultsLoaderFactory resultsLoaderFactory;
-
+ private Timer cpRelationTimer;
public CpRelationManager() {}
@@ -198,7 +200,8 @@ public class CpRelationManager implements RelationManager {
CpEntityManagerFactory emf,
UUID applicationId,
EntityRef headEntity,
- IndexBucketLocator indexBucketLocator ) {
+ IndexBucketLocator indexBucketLocator,
+ MetricsFactory metricsFactory) {
Assert.notNull( em, "Entity manager cannot be null" );
Assert.notNull( emf, "Entity manager factory cannot be null" );
@@ -219,6 +222,8 @@ public class CpRelationManager implements RelationManager {
this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
this.indexBucketLocator = indexBucketLocator; // TODO: this also
+ this.cpRelationTimer = metricsFactory.getTimer( CpRelationManager.class, "relation.manager.timer" );
+
// load the Core Persistence version of the head entity as well
this.headEntityScope = getCollectionScopeNameFromEntityType(
applicationScope.getApplication(), headEntity.getType());
@@ -441,8 +446,10 @@ public class CpRelationManager implements RelationManager {
}
} ).count().toBlocking().lastOrDefault( 0 );
-
+ //Adding graphite metrics
+ Timer.Context timeElasticIndexBatch = cpRelationTimer.time();
entityIndexBatch.execute();
+ timeElasticIndexBatch.stop();
logger.debug( "updateContainingCollectionsAndCollections() updated {} indexes", count );
}
@@ -1065,7 +1072,11 @@ public class CpRelationManager implements RelationManager {
Keyspace ko = cass.getApplicationKeyspace( applicationId );
Mutator<ByteBuffer> m = createMutator( ko, be );
batchUpdateEntityConnection( m, false, connection, UUIDGenerator.newTimeUUID() );
+ //Added Graphite Metrics
+ Timer.Context timeElasticIndexBatch = cpRelationTimer.time();
batchExecute( m, CassandraService.RETRY_COUNT );
+ timeElasticIndexBatch.stop();
+
return connection;
}
@@ -1236,7 +1247,11 @@ public class CpRelationManager implements RelationManager {
Mutator<ByteBuffer> m = createMutator( ko, be );
batchUpdateEntityConnection(
m, true, ( ConnectionRefImpl ) connectionRef, UUIDGenerator.newTimeUUID() );
+
+ //Added Graphite Metrics
+ Timer.Context timeDeleteConnections = cpRelationTimer.time();
batchExecute( m, CassandraService.RETRY_COUNT );
+ timeDeleteConnections.stop();
EntityRef connectingEntityRef = connectionRef.getConnectingEntity(); // source
EntityRef connectedEntityRef = connectionRef.getConnectedEntity(); // target
@@ -1291,7 +1306,11 @@ public class CpRelationManager implements RelationManager {
//
// batch.deindex( allTypesIndexScope, targetEntity );
+ //Added Graphite Metrics
+ Timer.Context timeDeleteConnection = cpRelationTimer.time();
batch.execute();
+ timeDeleteConnection.stop();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d3b469ea/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
index 45c9784..174e0a4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraPersistenceUtils.java
@@ -327,7 +327,7 @@ public class CassandraPersistenceUtils {
return batch;
}
-
+ //No longer does retries
public static MutationResult batchExecute( Mutator<?> m, int retries ) {
return m.execute();
[14/31] incubator-usergrid git commit: Upgraded plexus utils and
fixes build concurrency issues with surefire
Posted by sf...@apache.org.
Upgraded plexus utils and fixes build concurrency issues with surefire
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2c54a2b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2c54a2b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2c54a2b6
Branch: refs/heads/USERGRID-405
Commit: 2c54a2b6fa4314f696f78f2b2cf11ebcaac03d1e
Parents: 10ec635
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 15:57:11 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 15:11:22 2015 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 7 +-
.../core/metrics/MetricsFactoryImpl.java | 3 +-
.../persistence/index/guice/IndexModule.java | 7 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 74 ++++++++++++++++++++
stack/pom.xml | 24 ++++---
stack/rest/pom.xml | 2 +-
stack/services/pom.xml | 7 +-
.../usergrid/management/OrganizationIT.java | 2 +-
8 files changed, 109 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 77ce82e..d30bedb 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -101,9 +101,14 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
+ </dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 2528070..773828d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit;
@Singleton
public class MetricsFactoryImpl implements MetricsFactory {
- private final MetricsFig metricsFig;
+
private MetricRegistry registry;
private GraphiteReporter graphiteReporter;
private JmxReporter jmxReporter;
@@ -44,7 +44,6 @@ public class MetricsFactoryImpl implements MetricsFactory {
@Inject
public MetricsFactoryImpl(MetricsFig metricsFig) {
- this.metricsFig = metricsFig;
registry = new MetricRegistry();
String metricsHost = metricsFig.getHost();
if(!metricsHost.equals("false")) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index d9a14c9..6fee17e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.index.guice;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
@@ -36,9 +38,8 @@ public class IndexModule extends AbstractModule {
// install our configuration
install(new GuicyFigModule(IndexFig.class));
- install(new FactoryModuleBuilder()
- .implement(EntityIndex.class, EsEntityIndexImpl.class)
- .build(EntityIndexFactory.class));
+
+ bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
new file mode 100644
index 0000000..4bf8b59
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexBufferProducer;
+import org.apache.usergrid.persistence.index.IndexFig;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Get index from factory, adds caching
+ */
+public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
+
+ private final IndexFig config;
+ private final EsProvider provider;
+ private final EsIndexCache indexCache;
+ private final IndexBufferProducer indexBatchBufferProducer;
+
+ private LoadingCache<ApplicationScope, EntityIndex> eiCache =
+ CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, EntityIndex>() {
+ public EntityIndex load( ApplicationScope scope ) {
+ return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache);
+ }
+ } );
+
+ @Inject
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
+ final IndexBufferProducer indexBatchBufferProducer ){
+ this.config = config;
+ this.provider = provider;
+ this.indexCache = indexCache;
+ this.indexBatchBufferProducer = indexBatchBufferProducer;
+ }
+
+ @Override
+ public EntityIndex createEntityIndex(final ApplicationScope appScope) {
+ try{
+ return eiCache.get(appScope);
+ }catch (ExecutionException ee){
+ throw new RuntimeException(ee);
+ }
+ }
+
+ @Override
+ public void invalidate() {
+ eiCache.invalidateAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 53755b3..25b2258 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -127,6 +127,7 @@
<metrics.version>3.0.0</metrics.version>
<rx.version>0.19.6</rx.version>
+ <surefire.plugin.artifactName>surefire-junit47</surefire.plugin.artifactName>
<surefire.plugin.version>2.18.1</surefire.plugin.version>
<powermock.version>1.6.1</powermock.version>
@@ -1503,8 +1504,8 @@
<forkCount>${usergrid.it.forkCount}</forkCount>
<reuseForks>${usergrid.it.reuseForks}</reuseForks>
<threadCount>${usergrid.it.forkCount}</threadCount>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}
- </argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <testFailureIgnore>false</testFailureIgnore>
</configuration>
<!-- TODO, we may need an exclusion. Appears to be a classloader bug
@@ -1515,12 +1516,19 @@
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>${surefire.plugin.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.maven.surfire</groupId>
- <artifactId>common-junit3</artifactId>
- </exclusion>
- </exclusions>
+
+ <!--<exclusions>-->
+ <!--<exclusion>-->
+ <!--<groupId>org.apache.maven.surfire</groupId>-->
+ <!--<artifactId>common-junit3</artifactId>-->
+ <!--</exclusion>-->
+ <!--</exclusions>-->
+ </dependency>
+ <!-- override plex utils, otherwise bug from above SO post happens-->
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
</dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 3ad58b4..5a856de 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -115,7 +115,7 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 46b0878..f5e8dd5 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -108,9 +108,14 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
+ </dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
index bd96434..5bf0c3d 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
@@ -90,7 +90,7 @@ public class OrganizationIT {
OrganizationInfo organization2 = setup.getMgmtSvc().getOrganizationForApplication( applicationId );
assertNotNull( organization2 );
- assertEquals( "wrong organization name", orgName, organization2.getName() );
+ assertEquals( "wrong organization name", organization.getOrganization().getName(), organization2.getName() );
boolean verified = setup.getMgmtSvc().verifyAdminUserPassword( organization.getOwner().getUuid(), "test" );
assertTrue( verified );
[21/31] incubator-usergrid git commit: move timers up
Posted by sf...@apache.org.
move timers up
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5cbfafe6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5cbfafe6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5cbfafe6
Branch: refs/heads/USERGRID-405
Commit: 5cbfafe661867047e642cb34f31fc54a11893cf0
Parents: ef550fa
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 17:28:47 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 17:28:47 2015 -0700
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 73 +++++++++++++-------
1 file changed, 48 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5cbfafe6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 2cbe51f..da1217c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -25,6 +25,7 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.inject.Inject;
import com.google.inject.assistedinject.Assisted;
+import com.yammer.metrics.core.Clock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.usergrid.persistence.core.future.BetterFuture;
@@ -95,6 +96,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final IndexIdentifier.IndexAlias alias;
private final IndexIdentifier indexIdentifier;
private final IndexBufferProducer indexBatchBufferProducer;
+ private final Timer addTimer;
+ private final Timer addWriteAliasTimer;
+ private final Timer addReadAliasTimer;
+ private final Timer searchTimer;
/**
* We purposefully make this per instance. Some indexes may work, while others may fail
@@ -125,6 +130,13 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private static final MatchAllQueryBuilder MATCH_ALL_QUERY_BUILDER = QueryBuilders.matchAllQuery();
private EsIndexCache aliasCache;
+ private Timer removeAliasTimer;
+ private Timer mappingTimer;
+ private Timer refreshTimer;
+ private Timer cursorTimer;
+ private Timer getVersionsTimer;
+ private Timer allVersionsTimer;
+ private Timer deletePreviousTimer;
// private final Timer indexTimer;
@@ -139,11 +151,33 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.esProvider = provider;
this.config = config;
this.cursorTimeout = config.getQueryCursorTimeout();
- this.indexIdentifier = IndexingUtils.createIndexIdentifier( config, appScope );
+ this.indexIdentifier = IndexingUtils.createIndexIdentifier(config, appScope);
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
this.metricsFactory = metricsFactory;
+ this.addTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" );
+ this.removeAliasTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" );
+ this.addReadAliasTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" );
+ this.addWriteAliasTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" );
+ this.mappingTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" );
+ this.refreshTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" );
+ this.searchTimer =metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" );
+ this.cursorTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" );
+ this.getVersionsTimer =metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" );
+ this.allVersionsTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" );
+ this.deletePreviousTimer = metricsFactory
+ .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" );
}
@Override
@@ -175,8 +209,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.build();
//Added For Graphite Metrics
- Timer.Context timeNewIndexCreation = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.index.timer" ).time();
+ Timer.Context timeNewIndexCreation = addTimer.time();
final CreateIndexResponse cir = admin.indices().prepareCreate(indexName)
.setSettings(settings)
.execute()
@@ -209,8 +242,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
for (String currentIndex : indexNames){
//Added For Graphite Metrics
- Timer.Context timeRemoveAlias = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.remove.index.alias.timer" ).time();
+ Timer.Context timeRemoveAlias = removeAliasTimer.time();
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
alias.getWriteAlias()).execute().actionGet().isAcknowledged();
timeRemoveAlias.stop();
@@ -218,8 +250,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
//Added For Graphite Metrics
- Timer.Context timeAddReadAlias = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.read.alias.timer" ).time();
+ Timer.Context timeAddReadAlias = addReadAliasTimer.time();
// add read alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
@@ -227,8 +258,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
logger.info("Created new read Alias Name [{}] ACK=[{}]", alias.getReadAlias(), isAck);
//Added For Graphite Metrics
- Timer.Context timeAddWriteAlias = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.add.write.alias.timer" ).time();
+ Timer.Context timeAddWriteAlias = addWriteAliasTimer.time();
//add write alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
@@ -299,8 +329,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
XContentFactory.jsonBuilder(), "_default_");
//Added For Graphite Metrics
- Timer.Context timePutIndex = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.create.mapping.timer" ).time();
+ Timer.Context timePutIndex = mappingTimer.time();
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
// set mapping as the default for all types
@@ -393,8 +422,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
- Timer.Context timeSearch = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.timer" ).time();
+ Timer.Context timeSearch = searchTimer.time();
searchResponse = srb.execute().actionGet();
timeSearch.stop();
}
@@ -422,8 +450,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
- Timer.Context timeSearchCursor = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.search.cursor.timer" ).time();
+ Timer.Context timeSearchCursor = cursorTimer.time();
searchResponse = ssrb.execute().actionGet();
timeSearchCursor.stop();
}
@@ -495,8 +522,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return true;
}
//Added For Graphite Metrics
- Timer.Context timeRefreshIndex = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.refresh.timer" ).time();
+ Timer.Context timeRefreshIndex = refreshTimer.time();
esProvider.getClient().admin().indices().prepareRefresh( indexes ).execute().actionGet();
timeRefreshIndex.stop();
logger.debug("Refreshed indexes: {}", StringUtils.join(indexes, ", "));
@@ -542,8 +568,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final SearchResponse searchResponse;
try {
//Added For Graphite Metrics
- Timer.Context timeEntityIndex = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.get.versions.timer" ).time();
+ Timer.Context timeEntityIndex = getVersionsTimer.time();
searchResponse = srb.execute().actionGet();
timeEntityIndex.stop();
}
@@ -568,18 +593,17 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
//Added For Graphite Metrics
- final Timer.Context timeDeleteAllVersions = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.all.versions.timer" ).time();
+ final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
response.addListener( new ActionListener<DeleteByQueryResponse>() {
@Override
- public void onResponse( DeleteByQueryResponse response ) {
+ public void onResponse( DeleteByQueryResponse response) {
timeDeleteAllVersions.stop();
logger
- .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
+ .debug("Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
entityId.getUuid(), response.status().toString() );
checkDeleteByQueryResponse( tqb, response );
@@ -610,8 +634,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//Added For Graphite Metrics
//Checks the time from the execute to the response below
- final Timer.Context timeDeletePreviousVersions = metricsFactory
- .getTimer( EsEntityIndexImpl.class, "es.entity.index.delete.previous.versions.timer" ).time();
+ final Timer.Context timeDeletePreviousVersions = deletePreviousTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
[04/31] incubator-usergrid git commit: Added ES query and index
metrics.
Posted by sf...@apache.org.
Added ES query and index metrics.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8d378683
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8d378683
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8d378683
Branch: refs/heads/USERGRID-405
Commit: 8d37868338b67a6d4e22d96d7316b1b94bc61837
Parents: 00e7ca4
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 4 15:45:57 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 4 15:45:57 2015 -0800
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 82 ++++++++++++++------
1 file changed, 59 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8d378683/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 4200d13..4d4dce9 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -109,6 +109,8 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final IndexFig config;
+ private final MetricsFactory metricsFactory;
+
//number of times to wait for the index to refresh properly.
private static final int MAX_WAITS = 10;
@@ -126,9 +128,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private final Timer flushTimer;
- //private final Meter flushMeter;
-
-
@Inject
public EsEntityIndexImpl( @Assisted final ApplicationScope appScope, final IndexFig config,
@@ -144,6 +143,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.alias = indexIdentifier.getAlias();
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
+ this.metricsFactory = metricsFactory;
this.flushTimer = metricsFactory.getTimer( EsEntityIndexImpl.class, "entity.index.flush" );
}
@@ -175,6 +175,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.put("action.write_consistency", writeConsistency )
.build();
+ //Added For Graphite Metrics
Timer.Context timeNewIndexCreation = flushTimer.time();
final CreateIndexResponse cir = admin.indices().prepareCreate(indexName)
.setSettings(settings)
@@ -207,20 +208,28 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
String[] indexNames = getIndexes(AliasType.Write);
for (String currentIndex : indexNames){
+ //Added For Graphite Metrics
+ Timer.Context timeRemoveAlias = flushTimer.time();
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
alias.getWriteAlias()).execute().actionGet().isAcknowledged();
-
+ timeRemoveAlias.stop();
logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck);
}
+ //Added For Graphite Metrics
+ Timer.Context timeAddReadAlias = flushTimer.time();
// add read alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
+ timeAddReadAlias.stop();
logger.info("Created new read Alias Name [{}] ACK=[{}]", alias.getReadAlias(), isAck);
- // add write alias
+ //Added For Graphite Metrics
+ Timer.Context timeAddWriteAlias = flushTimer.time();
+ //add write alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
+ timeAddWriteAlias.stop();
logger.info("Created new write Alias Name [{}] ACK=[{}]", alias.getWriteAlias(), isAck);
aliasCache.invalidate(alias);
@@ -286,12 +295,14 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
XContentFactory.jsonBuilder(), "_default_");
+ //Added For Graphite Metrics
+ Timer.Context timePutIndex = flushTimer.time();
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
// set mapping as the default for all types
.setTemplate(config.getIndexPrefix() + "*").addMapping( "_default_", xcb )
.execute().actionGet();
-
+ timePutIndex.stop();
if ( !pitr.isAcknowledged() ) {
throw new IndexException( "Unable to create default mappings" );
}
@@ -301,7 +312,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
@Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
- applicationScope, esProvider.getClient(),indexBatchBufferProducer, config, this );
+ applicationScope, esProvider.getClient(),indexBatchBufferProducer, config, this, metricsFactory );
return batch;
}
@@ -377,7 +388,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearch = flushTimer.time();
searchResponse = srb.execute().actionGet();
+ timeSearch.stop();
}
catch ( Throwable t ) {
logger.error( "Unable to communicate with Elasticsearch", t );
@@ -402,7 +416,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.prepareSearchScroll(scrollId).setScroll( cursorTimeout + "m" );
try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearchCursor = flushTimer.time();
searchResponse = ssrb.execute().actionGet();
+ timeSearchCursor.stop();
}
catch ( Throwable t ) {
logger.error( "Unable to communicate with elasticsearch", t );
@@ -471,7 +488,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
applicationScope.getApplication().getUuid() );
return true;
}
+ //Added For Graphite Metrics
+ Timer.Context timeRefreshIndex = flushTimer.time();
esProvider.getClient().admin().indices().prepareRefresh( indexes ).execute().actionGet();
+ timeRefreshIndex.stop();
logger.debug("Refreshed indexes: {}", StringUtils.join(indexes, ", "));
return true;
}
@@ -514,7 +534,10 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final SearchResponse searchResponse;
try {
+ //Added For Graphite Metrics
+ Timer.Context timeEntityIndex = flushTimer.time();
searchResponse = srb.execute().actionGet();
+ timeEntityIndex.stop();
}
catch ( Throwable t ) {
logger.error( "Unable to communicate with elasticsearch" );
@@ -536,22 +559,28 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
+ //Added For Graphite Metrics
+ final Timer.Context timeDeleteAllVersions = flushTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
- response.addListener(new ActionListener<DeleteByQueryResponse>() {
+ response.addListener( new ActionListener<DeleteByQueryResponse>() {
+
@Override
- public void onResponse(DeleteByQueryResponse response) {
- logger.debug( "Deleted entity {}:{} from all index scopes with response status = {}",
- entityId.getType(), entityId.getUuid(), response.status().toString());
+ public void onResponse( DeleteByQueryResponse response ) {
+ timeDeleteAllVersions.stop();
+ logger
+ .debug( "Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
+ entityId.getUuid(), response.status().toString() );
checkDeleteByQueryResponse( tqb, response );
}
+
@Override
- public void onFailure(Throwable e) {
- logger.error("Deleted entity {}:{} from all index scopes with error {}",
- entityId.getType(), entityId.getUuid(), e);
+ public void onFailure( Throwable e ) {
+ logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+ entityId.getUuid(), e);
}
@@ -570,26 +599,33 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
FilterBuilders.rangeFilter(ENTITY_VERSION_FIELDNAME).lt(version.timestamp())
);
+ //Added For Graphite Metrics
+ //Checks the time from the execute to the response below
+ final Timer.Context timeDeletePreviousVersions = flushTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
- response.addListener(new ActionListener<DeleteByQueryResponse>() {
+ //Added For Graphite Metrics
+ response.addListener( new ActionListener<DeleteByQueryResponse>() {
@Override
- public void onResponse(DeleteByQueryResponse response) {
+ public void onResponse( DeleteByQueryResponse response ) {
+ timeDeletePreviousVersions.stop();
//error message needs to be retooled so that it describes the entity more throughly
- logger.debug( "Deleted entity {}:{} with version {} from all "
- + "index scopes with response status = {}",
- entityId.getType(), entityId.getUuid(), version, response.status().toString() );
+ logger
+ .debug( "Deleted entity {}:{} with version {} from all " + "index scopes with response status = {}",
+ entityId.getType(), entityId.getUuid(), version, response.status().toString() );
checkDeleteByQueryResponse( fqb, response );
}
+
@Override
- public void onFailure(Throwable e) {
- logger.error("Deleted entity {}:{} from all index scopes with error {}",
- entityId.getType(), entityId.getUuid(), e);
+ public void onFailure( Throwable e ) {
+ logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+ entityId.getUuid(), e );
}
- });
+ } );
+
return response;
}
[20/31] incubator-usergrid git commit: move timers up
Posted by sf...@apache.org.
move timers up
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ef550fa3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ef550fa3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ef550fa3
Branch: refs/heads/USERGRID-405
Commit: ef550fa380c4aa3422e907ad024194ad9bb29a6f
Parents: d542909
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 17:20:32 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 17:20:32 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 23 +++++++++++++-------
1 file changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef550fa3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index f5d59d9..d018324 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -190,6 +190,10 @@ public class CpRelationManager implements RelationManager {
private ResultsLoaderFactory resultsLoaderFactory;
private MetricsFactory metricsFactory;
+ private Timer updateCollectionTimer;
+ private Timer createConnectionTimer;
+ private Timer cassConnectionDelete;
+ private Timer esDeleteConnectionTimer;
public CpRelationManager() {}
@@ -221,7 +225,13 @@ public class CpRelationManager implements RelationManager {
this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
this.indexBucketLocator = indexBucketLocator; // TODO: this also
this.metricsFactory = metricsFactory;
-
+ this.updateCollectionTimer = metricsFactory
+ .getTimer( CpRelationManager.class, "relation.manager.es.update.collection" );
+ this.createConnectionTimer = metricsFactory
+ .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" );
+ this.cassConnectionDelete = metricsFactory
+ .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" );
+ this.esDeleteConnectionTimer = metricsFactory.getTimer(CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" );
// load the Core Persistence version of the head entity as well
this.headEntityScope = getCollectionScopeNameFromEntityType(
applicationScope.getApplication(), headEntity.getType());
@@ -445,8 +455,7 @@ public class CpRelationManager implements RelationManager {
} ).count().toBlocking().lastOrDefault( 0 );
//Adding graphite metrics
- Timer.Context timeElasticIndexBatch = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.es.update.collection" ).time();
+ Timer.Context timeElasticIndexBatch = updateCollectionTimer.time();
entityIndexBatch.execute();
timeElasticIndexBatch.stop();
@@ -1072,8 +1081,7 @@ public class CpRelationManager implements RelationManager {
Mutator<ByteBuffer> m = createMutator( ko, be );
batchUpdateEntityConnection( m, false, connection, UUIDGenerator.newTimeUUID() );
//Added Graphite Metrics
- Timer.Context timeElasticIndexBatch = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.es.create.connection.timer" ).time();
+ Timer.Context timeElasticIndexBatch = createConnectionTimer.time();
batchExecute( m, CassandraService.RETRY_COUNT );
timeElasticIndexBatch.stop();
@@ -1249,8 +1257,7 @@ public class CpRelationManager implements RelationManager {
m, true, ( ConnectionRefImpl ) connectionRef, UUIDGenerator.newTimeUUID() );
//Added Graphite Metrics
- Timer.Context timeDeleteConnections = metricsFactory
- .getTimer( CpRelationManager.class, "relation.manager.cassandra.delete.connection.batch.timer" ).time();
+ Timer.Context timeDeleteConnections = cassConnectionDelete.time();
batchExecute( m, CassandraService.RETRY_COUNT );
timeDeleteConnections.stop();
@@ -1308,7 +1315,7 @@ public class CpRelationManager implements RelationManager {
// batch.deindex( allTypesIndexScope, targetEntity );
//Added Graphite Metrics
- Timer.Context timeDeleteConnection = metricsFactory.getTimer( CpRelationManager.class, "relation.manager.es.delete.connection.batch.timer" ).time();
+ Timer.Context timeDeleteConnection = esDeleteConnectionTimer.time();
batch.execute();
timeDeleteConnection.stop();
[07/31] incubator-usergrid git commit: Added Metrics factory to
CpEntityManager and its corresponding factory.
Posted by sf...@apache.org.
Added Metrics factory to CpEntityManager and its corresponding factory.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/b3d4eb2b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/b3d4eb2b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/b3d4eb2b
Branch: refs/heads/USERGRID-405
Commit: b3d4eb2b50336307fdece0fc2edc390c4b658fcd
Parents: ff46465
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 4 16:06:17 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 4 16:06:17 2015 -0800
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/CpEntityManager.java | 10 +++++++++-
.../usergrid/corepersistence/CpEntityManagerFactory.java | 11 ++++++++---
2 files changed, 17 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3d4eb2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5619382..10e7926 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -72,6 +72,7 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.exception.WriteOptimisticVerifyException;
import org.apache.usergrid.persistence.collection.exception.WriteUniqueVerifyException;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.entities.Application;
@@ -103,6 +104,7 @@ import org.apache.usergrid.utils.CompositeUtils;
import org.apache.usergrid.utils.StringUtils;
import org.apache.usergrid.utils.UUIDUtils;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.netflix.hystrix.exception.HystrixRuntimeException;
import com.yammer.metrics.annotation.Metered;
@@ -187,12 +189,16 @@ public class CpEntityManager implements EntityManager {
private boolean skipAggregateCounters;
+ private final Timer corePersistanceTimer;
+
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
- public CpEntityManager() {}
+ public CpEntityManager() {
+ corePersistanceTimer = emf.getMetricsFactory().getTimer( CpEntityManager.class, "cp.entity.manager.timer" );
+ }
@Override
public void init( EntityManagerFactory emf, UUID applicationId ) {
@@ -303,7 +309,9 @@ public class CpEntityManager implements EntityManager {
Entity entity = batchCreate( m, entityType, null, properties, importId, timestampUuid );
+ Timer.Context timeCassCreation = corePersistanceTimer.time();
m.execute();
+ timeCassCreation.stop();
return entity;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/b3d4eb2b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9aa36a0..b247eae 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -44,6 +44,7 @@ import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.CollectionScope;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
+import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.DataMigrationManager;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
@@ -105,17 +106,18 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private CassandraService cassandraService;
private CounterUtils counterUtils;
private Injector injector;
-
+ private final MetricsFactory metricsFactory;
public CpEntityManagerFactory(
- final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector) {
+ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector,
+ final MetricsFactory metricsFactory) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
this.injector = injector;
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-
+ this.metricsFactory = metricsFactory;
}
@@ -194,6 +196,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
return em;
}
+ public MetricsFactory getMetricsFactory(){
+ return metricsFactory;
+ }
@Override
public UUID createApplication(String organizationName, String name) throws Exception {
[23/31] incubator-usergrid git commit: Merge remote-tracking branch
'origin/USERGRID-454' into USERGRID-416
Posted by sf...@apache.org.
Merge remote-tracking branch 'origin/USERGRID-454' into USERGRID-416
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/729c675f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/729c675f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/729c675f
Branch: refs/heads/USERGRID-405
Commit: 729c675f11aac1c0f77e268df4ccbfee126334f7
Parents: b8f5bf6 5e0ce62
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 17:49:20 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 17:49:20 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 108 ++++++++++++++--
.../corepersistence/CpEntityManagerFactory.java | 8 +-
.../corepersistence/CpRelationManager.java | 39 +++++-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
.../index/impl/EsEntityIndexBatchImpl.java | 15 ++-
.../index/impl/EsEntityIndexImpl.java | 125 +++++++++++++++----
6 files changed, 257 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/729c675f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 652e084,f7a9f8b..e79ec83
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@@ -676,9 -692,8 +692,9 @@@ public class CpEntityManager implement
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
+ Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager rmi = new CpRelationManager();
- rmi.init( this, emf, applicationId, entityRef, null );
+ rmi.init( this, emf, applicationId, entityRef, null, metricsFactory );
return rmi;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/729c675f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e0029c1,3f439e6..18c7965
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -118,8 -117,8 +120,9 @@@ public class CpEntityManagerFactory imp
this.injector = injector;
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
+ this.metricsFactory = injector.getInstance( MetricsFactory.class );
+ this.orgApplicationCache = new OrgApplicationCacheImpl( this );
}
[16/31] incubator-usergrid git commit: changing buffer impl to queue
Posted by sf...@apache.org.
changing buffer impl to queue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5ad1a8c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5ad1a8c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5ad1a8c9
Branch: refs/heads/USERGRID-405
Commit: 5ad1a8c953fc3b8a6b48e6d0ca3c7c0da52ec8d1
Parents: 615a5af
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 6 15:47:28 2015 -0700
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 6 15:47:28 2015 -0700
----------------------------------------------------------------------
.../persistence/index/IndexBufferProducer.java | 10 +++--
.../usergrid/persistence/index/IndexFig.java | 12 +++++-
.../index/impl/EsIndexBufferConsumerImpl.java | 41 +++++++++++++++++++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 +++++++++-----
.../persistence/index/impl/EntityIndexTest.java | 2 +-
5 files changed, 77 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5ad1a8c9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
index 6338a0c..19d224c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexBufferProducer.java
@@ -24,13 +24,15 @@ import org.apache.usergrid.persistence.index.IndexOperationMessage;
import rx.Observable;
import rx.Subscriber;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
/**
* Classy class class.
*/
-public interface IndexBufferProducer extends Observable.OnSubscribe<IndexOperationMessage> {
-
- @Override
- void call(Subscriber<? super IndexOperationMessage> subscriber);
+public interface IndexBufferProducer {
BetterFuture put(IndexOperationMessage message);
+
+ BlockingQueue<IndexOperationMessage> getSource();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5ad1a8c9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index 9893ca5..c6f08f6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -51,6 +51,8 @@ public interface IndexFig extends GuicyFig {
public static final String INDEX_BUFFER_SIZE = "elasticsearch.buffer_size";
+ public static final String INDEX_QUEUE_SIZE = "elasticsearch.queue_size";
+
public static final String INDEX_BUFFER_TIMEOUT = "elasticsearch.buffer_timeout";
public static final String INDEX_BATCH_SIZE = "elasticsearch.batch_size";
@@ -127,7 +129,7 @@ public interface IndexFig extends GuicyFig {
*/
@Default("250")
@Key( INDEX_BUFFER_TIMEOUT )
- int getIndexBufferTimeout();
+ long getIndexBufferTimeout();
/**
* size of the buffer to build up before you send results
@@ -138,6 +140,14 @@ public interface IndexFig extends GuicyFig {
int getIndexBufferSize();
/**
+ * size of the buffer to build up before you send results
+ * @return
+ */
+ @Default("1000")
+ @Key( INDEX_QUEUE_SIZE )
+ int getIndexQueueSize();
+
+ /**
* Request batch size for ES
* @return
*/
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5ad1a8c9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
index 52d1abb..625f4e7 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferConsumerImpl.java
@@ -42,12 +42,14 @@ import org.elasticsearch.client.Client;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
+import rx.Subscriber;
import rx.functions.Action1;
import rx.functions.Func1;
import rx.schedulers.Schedulers;
+import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.TimeUnit;
+import java.util.concurrent.*;
/**
* Consumer for IndexOperationMessages
@@ -63,6 +65,7 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
private final Timer flushTimer;
private final Counter indexSizeCounter;
private final Meter flushMeter;
+ private final Timer produceTimer;
@Inject
public EsIndexBufferConsumerImpl(final IndexFig config, final IndexBufferProducer producer, final EsProvider provider, final MetricsFactory metricsFactory){
@@ -72,14 +75,48 @@ public class EsIndexBufferConsumerImpl implements IndexBufferConsumer {
this.config = config;
this.failureMonitor = new FailureMonitorImpl(config,provider);
this.client = provider.getClient();
+ this.produceTimer = metricsFactory.getTimer(EsIndexBufferConsumerImpl.class,"index.buffer.consumer.messageFetch");
+ final BlockingQueue<IndexOperationMessage> producerQueue = producer.getSource();
//batch up sets of some size and send them in batch
- this.consumer = Observable.create(producer)
+ this.consumer = Observable.create(new Observable.OnSubscribe<IndexOperationMessage>() {
+ @Override
+ public void call(final Subscriber<? super IndexOperationMessage> subscriber) {
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ List<IndexOperationMessage> drainList = new ArrayList<>(config.getIndexBufferSize() + 1);
+ do {
+ try {
+ Timer.Context timer = produceTimer.time();
+ IndexOperationMessage polled = producerQueue.poll(config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS);
+ if(polled!=null) {
+ drainList.add(polled);
+ producerQueue.drainTo(drainList, config.getIndexBufferSize());
+ System.out.println("Consumer Thread" + Thread.currentThread().getName());
+ for(IndexOperationMessage drained : drainList){
+ subscriber.onNext(drained);
+ }
+ drainList.clear();
+ }
+ timer.stop();
+
+ } catch (InterruptedException ie) {
+ log.error("failed to dequeue", ie);
+ }
+ } while (true);
+ }
+ });
+ thread.setName("EsEntityIndex_Consumer");
+ thread.start();
+ }
+ })
.subscribeOn(Schedulers.io())
.buffer(config.getIndexBufferTimeout(), TimeUnit.MILLISECONDS, config.getIndexBufferSize())
.doOnNext(new Action1<List<IndexOperationMessage>>() {
@Override
public void call(List<IndexOperationMessage> containerList) {
+ System.out.println("Buffered Consumer Thread" + Thread.currentThread().getName());
if (containerList.size() > 0) {
flushMeter.mark(containerList.size());
Timer.Context time = flushTimer.time();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5ad1a8c9/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
index 29f243b..d4d621f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexBufferProducerImpl.java
@@ -20,15 +20,20 @@
package org.apache.usergrid.persistence.index.impl;
import com.codahale.metrics.Counter;
+import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
import com.google.inject.Inject;
import com.google.inject.Singleton;
import org.apache.usergrid.persistence.core.future.BetterFuture;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.index.IndexBufferProducer;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexOperationMessage;
import rx.Subscriber;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
/**
* Producer for index operation messages
*/
@@ -36,22 +41,27 @@ import rx.Subscriber;
public class EsIndexBufferProducerImpl implements IndexBufferProducer {
private final Counter indexSizeCounter;
- private Subscriber<? super IndexOperationMessage> subscriber;
+ private final ArrayBlockingQueue<IndexOperationMessage> messages;
+ private final Timer timer;
@Inject
- public EsIndexBufferProducerImpl(MetricsFactory metricsFactory){
- this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferProducerImpl.class,"index.buffer.size");
-
- }
- @Override
- public void call(Subscriber<? super IndexOperationMessage> subscriber) {
- this.subscriber = subscriber;
+ public EsIndexBufferProducerImpl(MetricsFactory metricsFactory,IndexFig fig){
+ this.messages = new ArrayBlockingQueue<>(fig.getIndexQueueSize()*5);
+ this.indexSizeCounter = metricsFactory.getCounter(EsIndexBufferProducerImpl.class, "index.buffer.size");
+ this.timer = metricsFactory.getTimer(EsIndexBufferProducerImpl.class,"index.buffer.producer.timer");
}
public BetterFuture put(IndexOperationMessage message){
- Preconditions.checkNotNull(message,"Message cannot be null");
+ Preconditions.checkNotNull(message, "Message cannot be null");
indexSizeCounter.inc(message.getOperations().size());
- subscriber.onNext(message);
+ Timer.Context time = timer.time();
+ messages.offer(message);
+ time.stop();
return message.getFuture();
}
+
+ @Override
+ public BlockingQueue<IndexOperationMessage> getSource() {
+ return messages;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5ad1a8c9/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 22aadc0..70ae8c5 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -96,7 +96,7 @@ public class EntityIndexTest extends BaseIT {
long now = System.currentTimeMillis();
final int threads = 20;
- final int size = 20;
+ final int size = 30;
final EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
final IndexScope indexScope = new IndexScopeImpl(appId, "things");
final String entityType = "thing";
[09/31] incubator-usergrid git commit: Fixed wiring issues present
when running tests.
Posted by sf...@apache.org.
Fixed wiring issues present when running tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cb35127f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cb35127f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cb35127f
Branch: refs/heads/USERGRID-405
Commit: cb35127f30ea4d155ce93577b8a27547f93223fd
Parents: d3b469e
Author: GERey <gr...@apigee.com>
Authored: Thu Mar 5 12:32:07 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Thu Mar 5 12:32:07 2015 -0800
----------------------------------------------------------------------
.../org/apache/usergrid/corepersistence/CpEntityManager.java | 6 +++---
.../usergrid/corepersistence/CpEntityManagerFactory.java | 6 +++---
.../org/apache/usergrid/corepersistence/CpRelationManager.java | 6 ++++--
3 files changed, 10 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb35127f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 4355638..e7efa6f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -189,14 +189,13 @@ public class CpEntityManager implements EntityManager {
private boolean skipAggregateCounters;
- private final Timer corePersistanceTimer;
+ private Timer corePersistanceTimer;
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
public CpEntityManager() {
- corePersistanceTimer = emf.getMetricsFactory().getTimer( CpEntityManager.class, "cp.entity.manager.timer" );
}
@@ -207,6 +206,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull( applicationId, "applicationId must not be null" );
this.emf = ( CpEntityManagerFactory ) emf;
+ corePersistanceTimer = this.emf.getMetricsFactory().getTimer( CpEntityManager.class, "cp.entity.manager.timer" );
this.managerCache = this.emf.getManagerCache();
this.applicationId = applicationId;
@@ -689,7 +689,7 @@ public class CpEntityManager implements EntityManager {
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
CpRelationManager rmi = new CpRelationManager();
- rmi.init( this, emf, applicationId, entityRef, null );
+ rmi.init( this, emf, applicationId, entityRef, null, emf.getMetricsFactory() );
return rmi;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb35127f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b247eae..3f439e6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.corepersistence;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.inject.Guice;
import com.google.inject.Injector;
import com.yammer.metrics.annotation.Metered;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
@@ -109,15 +110,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final MetricsFactory metricsFactory;
public CpEntityManagerFactory(
- final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector,
- final MetricsFactory metricsFactory) {
+ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
this.injector = injector;
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
- this.metricsFactory = metricsFactory;
+ this.metricsFactory = injector.getInstance( MetricsFactory.class );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb35127f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 786b0f5..499f841 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -192,6 +192,8 @@ public class CpRelationManager implements RelationManager {
private Timer cpRelationTimer;
+ private MetricsFactory metricsFactory;
+
public CpRelationManager() {}
@@ -221,7 +223,7 @@ public class CpRelationManager implements RelationManager {
this.cass = em.getCass(); // TODO: eliminate need for this via Core Persistence
this.indexBucketLocator = indexBucketLocator; // TODO: this also
-
+ this.metricsFactory = metricsFactory;
this.cpRelationTimer = metricsFactory.getTimer( CpRelationManager.class, "relation.manager.timer" );
// load the Core Persistence version of the head entity as well
@@ -1541,7 +1543,7 @@ public class CpRelationManager implements RelationManager {
private CpRelationManager getRelationManager( EntityRef headEntity ) {
CpRelationManager rmi = new CpRelationManager();
- rmi.init( em, emf, applicationId, headEntity, null );
+ rmi.init( em, emf, applicationId, headEntity, null, metricsFactory);
return rmi;
}
[05/31] incubator-usergrid git commit: Changed timer name to be more
descriptive
Posted by sf...@apache.org.
Changed timer name to be more descriptive
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0f203b67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0f203b67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0f203b67
Branch: refs/heads/USERGRID-405
Commit: 0f203b67fb3f0f2d1525ac24ce51b47232931937
Parents: 8d37868
Author: GERey <gr...@apigee.com>
Authored: Wed Mar 4 15:47:01 2015 -0800
Committer: GERey <gr...@apigee.com>
Committed: Wed Mar 4 15:47:01 2015 -0800
----------------------------------------------------------------------
.../index/impl/EsEntityIndexImpl.java | 26 ++++++++++----------
1 file changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0f203b67/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 4d4dce9..f85692f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -126,7 +126,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
private EsIndexCache aliasCache;
- private final Timer flushTimer;
+ private final Timer indexTimer;
@Inject
@@ -144,7 +144,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
this.failureMonitor = new FailureMonitorImpl( config, provider );
this.aliasCache = indexCache;
this.metricsFactory = metricsFactory;
- this.flushTimer = metricsFactory.getTimer( EsEntityIndexImpl.class, "entity.index.flush" );
+ this.indexTimer = metricsFactory.getTimer( EsEntityIndexImpl.class, "entity.index.timer" );
}
@Override
@@ -176,7 +176,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.build();
//Added For Graphite Metrics
- Timer.Context timeNewIndexCreation = flushTimer.time();
+ Timer.Context timeNewIndexCreation = indexTimer.time();
final CreateIndexResponse cir = admin.indices().prepareCreate(indexName)
.setSettings(settings)
.execute()
@@ -209,7 +209,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
for (String currentIndex : indexNames){
//Added For Graphite Metrics
- Timer.Context timeRemoveAlias = flushTimer.time();
+ Timer.Context timeRemoveAlias = indexTimer.time();
isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
alias.getWriteAlias()).execute().actionGet().isAcknowledged();
timeRemoveAlias.stop();
@@ -217,7 +217,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
//Added For Graphite Metrics
- Timer.Context timeAddReadAlias = flushTimer.time();
+ Timer.Context timeAddReadAlias = indexTimer.time();
// add read alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
@@ -225,7 +225,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
logger.info("Created new read Alias Name [{}] ACK=[{}]", alias.getReadAlias(), isAck);
//Added For Graphite Metrics
- Timer.Context timeAddWriteAlias = flushTimer.time();
+ Timer.Context timeAddWriteAlias = indexTimer.time();
//add write alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getWriteAlias()).execute().actionGet().isAcknowledged();
@@ -296,7 +296,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
XContentFactory.jsonBuilder(), "_default_");
//Added For Graphite Metrics
- Timer.Context timePutIndex = flushTimer.time();
+ Timer.Context timePutIndex = indexTimer.time();
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
// set mapping as the default for all types
@@ -389,7 +389,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
- Timer.Context timeSearch = flushTimer.time();
+ Timer.Context timeSearch = indexTimer.time();
searchResponse = srb.execute().actionGet();
timeSearch.stop();
}
@@ -417,7 +417,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
try {
//Added For Graphite Metrics
- Timer.Context timeSearchCursor = flushTimer.time();
+ Timer.Context timeSearchCursor = indexTimer.time();
searchResponse = ssrb.execute().actionGet();
timeSearchCursor.stop();
}
@@ -489,7 +489,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
return true;
}
//Added For Graphite Metrics
- Timer.Context timeRefreshIndex = flushTimer.time();
+ Timer.Context timeRefreshIndex = indexTimer.time();
esProvider.getClient().admin().indices().prepareRefresh( indexes ).execute().actionGet();
timeRefreshIndex.stop();
logger.debug("Refreshed indexes: {}", StringUtils.join(indexes, ", "));
@@ -535,7 +535,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final SearchResponse searchResponse;
try {
//Added For Graphite Metrics
- Timer.Context timeEntityIndex = flushTimer.time();
+ Timer.Context timeEntityIndex = indexTimer.time();
searchResponse = srb.execute().actionGet();
timeEntityIndex.stop();
}
@@ -560,7 +560,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
//Added For Graphite Metrics
- final Timer.Context timeDeleteAllVersions = flushTimer.time();
+ final Timer.Context timeDeleteAllVersions = indexTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
@@ -601,7 +601,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//Added For Graphite Metrics
//Checks the time from the execute to the response below
- final Timer.Context timeDeletePreviousVersions = flushTimer.time();
+ final Timer.Context timeDeletePreviousVersions = indexTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery(alias.getWriteAlias()).setQuery(fqb).execute();
[15/31] incubator-usergrid git commit: Added caching to org/app lookup
Posted by sf...@apache.org.
Added caching to org/app lookup
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c8727c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c8727c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c8727c1
Branch: refs/heads/USERGRID-405
Commit: 4c8727c1dc0e5f02a498849be1a19e6342f6accf
Parents: 2c54a2b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 15:13:49 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 15:13:49 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 1 +
.../corepersistence/CpEntityManagerFactory.java | 116 ++++--------
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++++
.../main/resources/usergrid-core-context.xml | 14 +-
.../persistence/index/EntityIndexFactory.java | 4 +-
.../resources/usergrid-services-context.xml | 8 +-
7 files changed, 296 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5619382..652e084 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -676,6 +676,7 @@ public class CpEntityManager implements EntityManager {
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
+ Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager rmi = new CpRelationManager();
rmi.init( this, emf, applicationId, entityRef, null );
return rmi;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9aa36a0..16de53e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,6 +15,7 @@
*/
package org.apache.usergrid.corepersistence;
+import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -98,6 +99,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
});
+ private final OrgApplicationCache orgApplicationCache;
+
private ManagerCache managerCache;
private DataMigrationManager dataMigrationManager;
@@ -116,7 +119,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-
+ this.orgApplicationCache = new OrgApplicationCacheImpl( this );
}
@@ -156,18 +159,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
-// public ManagerCache getManagerCache() {
-//
-// if ( managerCache == null ) {
-// managerCache = injector.getInstance( ManagerCache.class );
-//
-// dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-// }
-// return managerCache;
-// }
-
-
-
@Override
public EntityManager getEntityManager(UUID applicationId) {
try {
@@ -207,15 +198,16 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
String appName = buildAppName( orgName, name );
- UUID applicationId = lookupApplication( appName );
- if ( applicationId != null ) {
+ final Optional<UUID> appId = orgApplicationCache.getApplicationId( appName );
+
+ if ( appId.isPresent() ) {
throw new ApplicationAlreadyExistsException( name );
}
- applicationId = UUIDGenerator.newTimeUUID();
+ UUID applicationId = UUIDGenerator.newTimeUUID();
- logger.debug( "New application orgName {} name {} id {} ",
+ logger.debug( "New application orgName {} orgAppName {} id {} ",
new Object[] { orgName, name, applicationId.toString() } );
initializeApplication( orgName, applicationId, appName, properties );
@@ -233,6 +225,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Map<String, Object> properties ) throws Exception {
+
+ //Ensure our management system exists before creating our application
+ init();
+
EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
final String appName = buildAppName( organizationName, name );
@@ -244,8 +240,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
getSetup().setupApplicationKeyspace( applicationId, appName );
- UUID orgUuid = lookupOrganization( organizationName );
- if ( orgUuid == null ) {
+
+ final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( name );
+
+
+ final UUID orgUuid;
+
+ if ( !cachedValue.isPresent() ) {
+
// create new org because the specified one does not exist
final String orgName = organizationName;
@@ -263,6 +265,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.refreshIndex();
orgUuid = orgInfo.getUuid();
+
+ //evit so it's re-loaded later
+ orgApplicationCache.evictOrgId( name );
+ } else{
+ orgUuid = cachedValue.get();
}
// create appinfo entry in the system app
@@ -293,6 +300,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
appEm.refreshIndex();
logger.info("Initialized application {}", appName );
+
+ //evict app Id from cache
+ orgApplicationCache.evictAppId( appName );
+
return applicationId;
}
@@ -374,71 +385,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public UUID lookupOrganization( String name ) throws Exception {
- init();
-
-
- // Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-
-
- final EntityRef alias = em.getAlias( "organizations", name );
-
- if ( alias == null ) {
- return null;
- }
-
- final Entity entity = em.get( alias );
-
- if ( entity == null ) {
- return null;
- }
-
- return entity.getUuid();
- // Results results = em.searchCollection( em.getApplicationRef(), "organizations", q );
- //
- // if ( results.isEmpty() ) {
- // return null;
- // }
- //
- // return results.iterator().next().getUuid();
- }
-
@Override
- public UUID lookupApplication( String name ) throws Exception {
- init();
-
- // TODO: why does this not work for restored apps
-
-// EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-// final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name );
-// if ( alias == null ) {
-// return null;
-// }
-// final Entity entity = em.get( alias );
-// if ( entity == null ) {
-// return null;
-// }
-// final UUID property = ( UUID ) entity.getProperty( "applicationUuid" );
-// return property;
-
- Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
-
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
-
- Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
-
- if ( results.isEmpty() ) {
- return null;
- }
-
- Entity entity = results.iterator().next();
- Object uuidObject = entity.getProperty("applicationUuid");
- if ( uuidObject instanceof UUID ) {
- return (UUID)uuidObject;
- }
- return UUIDUtils.tryExtractUUID( entity.getProperty("applicationUuid").toString() );
+ public UUID lookupApplication( String orgAppName ) throws Exception {
+ return orgApplicationCache.getApplicationId( orgAppName ).orNull();
}
@@ -632,7 +582,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.update( propsEntity );
} catch (Exception ex) {
- logger.error("Error deleting service property name: " + name, ex);
+ logger.error("Error deleting service property orgAppName: " + name, ex);
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
new file mode 100644
index 0000000..b20dbe1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Entity;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A simple cache interface for looking up entities from an EM
+ */
+public interface OrgApplicationCache {
+
+
+ /**
+ * Get an entity by it's alias property. The result is cached. To clear it call evict or evict all
+ * @param
+ * @return
+ */
+ public Optional<UUID> getOrganizationId(final String orgName);
+
+ /**
+ * Evict the org by name
+ * @param orgName
+ */
+ public void evictOrgId(final String orgName);
+
+ /**
+ * Evict the application by name
+ * @param applicationName
+ * @return
+ */
+ public Optional<UUID> getApplicationId(final String applicationName);
+
+
+ /**
+ * Evict the app id by the name
+ */
+ public void evictAppId(final String applicationname);
+
+
+ /**
+ * Evict all caches
+ */
+ public void evictAll();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
new file mode 100644
index 0000000..4baf598
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+
+
+/**
+ * Implements the org app cache for faster runtime lookups. These values are immutable, so this LRU cache can stay
+ * full for the duration of the execution
+ */
+public class OrgApplicationCacheImpl implements OrgApplicationCache {
+
+
+ /**
+ * Cache the pointer to our root entity manager for reference
+ */
+ private final EntityManager rootEm;
+
+ private final LoadingCache<String, Optional<UUID>> orgCache =
+ CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String, Optional<UUID>>() {
+ @Override
+ public Optional<UUID> load( final String key ) throws Exception {
+ return fetchOrganizationId( key );
+ }
+ } );
+
+
+ private final LoadingCache<String, Optional<UUID>> appCache =
+ CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String, Optional<UUID>>() {
+ @Override
+ public Optional<UUID> load( final String key ) throws Exception {
+ return fetchApplicationId( key );
+ }
+ } );
+
+
+ public OrgApplicationCacheImpl( final EntityManagerFactory emf ) {
+ this.rootEm = emf.getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
+ }
+
+
+ @Override
+ public Optional<UUID> getOrganizationId( final String orgName ) {
+ try {
+ return orgCache.get( orgName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to load org cache", e );
+ }
+ }
+
+
+ /**
+ * Fetches the organization
+ */
+ private Optional<UUID> fetchOrganizationId( final String orgName ) {
+
+ try {
+ final EntityRef alias = rootEm.getAlias( "organizations", orgName );
+
+ if ( alias == null ) {
+ return Optional.absent();
+ }
+
+ final Entity entity;
+
+ entity = rootEm.get( alias );
+
+
+ if ( entity == null ) {
+ return Optional.absent();
+ }
+
+ return Optional.of( entity.getUuid() );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load organization Id for caching", e );
+ }
+ }
+
+
+ @Override
+ public void evictOrgId( final String orgName ) {
+ orgCache.invalidate( orgName );
+ }
+
+
+ @Override
+ public Optional<UUID> getApplicationId( final String applicationName ) {
+ try {
+ return appCache.get( applicationName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to load org cache", e );
+ }
+ }
+
+
+ /**
+ * Fetch our application id
+ */
+ private Optional<UUID> fetchApplicationId( final String applicationName ) {
+
+ try {
+ Query q = Query.fromQL( PROPERTY_NAME + " = '" + applicationName + "'" );
+
+
+ Results results = rootEm.searchCollection( rootEm.getApplicationRef(), "appinfos", q );
+
+ if ( results.isEmpty() ) {
+ return Optional.absent();
+ }
+
+ Entity entity = results.iterator().next();
+ Object uuidObject = entity.getProperty( "applicationUuid" );
+
+ final UUID value;
+ if ( uuidObject instanceof UUID ) {
+ value = ( UUID ) uuidObject;
+ }
+ else {
+ value = UUIDUtils.tryExtractUUID( entity.getProperty( "applicationUuid" ).toString() );
+ }
+
+
+ return Optional.of( value );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to retreive application id", e );
+ }
+ }
+
+
+ @Override
+ public void evictAppId( final String applicationName ) {
+ appCache.invalidate( applicationName );
+ }
+
+
+ @Override
+ public void evictAll() {
+ orgCache.invalidateAll();
+ appCache.invalidateAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index c8ce0d8..cd40d6d 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -54,7 +54,7 @@
<constructor-arg value="${cassandra.url}" />
<!-- set the pool size if it's available. If not go with 50 -->
<property name="maxActive" value="${cassandra.connections:50}"/>
- <!--<property name="clockResolution" ref="microsecondsTimeResolution" />-->
+ <!--<property orgAppName="clockResolution" ref="microsecondsTimeResolution" />-->
<property name="opTimer" ref="taggedOpTimer"/>
<property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
</bean>
@@ -70,7 +70,7 @@
<bean id="loadBalancingPolicy" class="me.prettyprint.cassandra.connection.DynamicLoadBalancingPolicy"/>
<!-- locking for a single node -->
-<!-- <bean name="lockManager"
+<!-- <bean orgAppName="lockManager"
class="org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl" />-->
<!-- hector based locks -->
@@ -86,10 +86,10 @@
<!-- zookeeper locks -->
<!--
- <bean name="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl" >
- <property name="hostPort" value="${zookeeper.url}"/>
- <property name="sessionTimeout" value="2000"/>
- <property name="maxAttempts" value="10"/>
+ <bean orgAppName="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl" >
+ <property orgAppName="hostPort" value="${zookeeper.url}"/>
+ <property orgAppName="sessionTimeout" value="2000"/>
+ <property orgAppName="maxAttempts" value="10"/>
</bean> -->
@@ -196,7 +196,7 @@
</bean>
<bean id="jobFactory" class="org.apache.usergrid.batch.UsergridJobFactory" />
-
+
<context:component-scan base-package="org.apache.usergrid.batch.job" />
<context:annotation-config />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
index 78a5137..10752d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
@@ -25,6 +25,8 @@ import com.google.inject.assistedinject.Assisted;
public interface EntityIndexFactory {
- public EntityIndex createEntityIndex(
+ public EntityIndex createEntityIndex(
@Assisted ApplicationScope appScope);
+
+ void invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index 666051d..8674b71 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -103,10 +103,10 @@
<!--<bean id="importQueueListener" class="org.apache.usergrid.services.queues.ImportQueueListener"-->
<!--scope="singleton">-->
- <!--<constructor-arg name="emf" ref="entityManagerFactory" />-->
- <!--<constructor-arg name="metricsService" ref="metricsFactory" />-->
- <!--<constructor-arg name="smf" ref="serviceManagerFactory" />-->
- <!--<constructor-arg name="props" ref="properties" />-->
+ <!--<constructor-arg orgAppName="emf" ref="entityManagerFactory" />-->
+ <!--<constructor-arg orgAppName="metricsService" ref="metricsFactory" />-->
+ <!--<constructor-arg orgAppName="smf" ref="serviceManagerFactory" />-->
+ <!--<constructor-arg orgAppName="props" ref="properties" />-->
<!--</bean>-->
[24/31] incubator-usergrid git commit: Merge remote-tracking branch
'origin/USERGRID-432' into USERGRID-416
Posted by sf...@apache.org.
Merge remote-tracking branch 'origin/USERGRID-432' into USERGRID-416
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/046ba4eb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/046ba4eb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/046ba4eb
Branch: refs/heads/USERGRID-405
Commit: 046ba4ebd884e02e4b34a7101a1b39504b174b36
Parents: 729c675 6066b6e
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 17:57:21 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 17:57:21 2015 -0700
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 ++++++++++++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 +++++++++++----
.../corepersistence/CpEntityManagerFactory.java | 11 +-----
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../persistence/index/IndexBufferProducer.java | 10 +++--
.../usergrid/persistence/index/IndexFig.java | 12 +++++-
.../index/impl/EsEntityIndexImpl.java | 32 +++++++++++-----
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++++++++++++++++++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++++++++++-----
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../services/notifications/wns/WNSAdapter.java | 39 ++++++++++++++++++++
13 files changed, 187 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/046ba4eb/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/046ba4eb/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --cc stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index da1217c,eac3822..951f96b
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@@ -240,17 -193,21 +241,30 @@@ public class EsEntityIndexImpl implemen
String[] indexNames = getIndexes(AliasType.Write);
- for (String currentIndex : indexNames) {
+ for (String currentIndex : indexNames){
++
++ final Timer.Context timeRemoveAlias = removeAliasTimer.time();
+ try {
- isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
+ //Added For Graphite Metrics
- Timer.Context timeRemoveAlias = removeAliasTimer.time();
++
+ isAck = adminClient.indices().prepareAliases().removeAlias(currentIndex,
alias.getWriteAlias()).execute().actionGet().isAcknowledged();
- timeRemoveAlias.stop();
+
- logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck);
+ logger.info("Removed Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, isAck);
+ } catch (AliasesMissingException aie) {
- logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, aie.getMessage());
- continue;
- }catch(InvalidAliasNameException iane) {
- logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, iane.getMessage());
- continue;
++ logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, aie.getMessage());
++ continue;
++ }catch(InvalidAliasNameException iane) {
++ logger.info("Alias does not exist Index Name [{}] from Alias=[{}] ACK=[{}]", currentIndex, alias, iane.getMessage());
++ continue;
++ }finally{
++ timeRemoveAlias.stop();
++
+ }
}
+ //Added For Graphite Metrics
+ Timer.Context timeAddReadAlias = addReadAliasTimer.time();
// add read alias
isAck = adminClient.indices().prepareAliases().addAlias(
indexName, alias.getReadAlias()).execute().actionGet().isAcknowledged();
@@@ -326,10 -279,8 +340,10 @@@
private void createMappings() throws IOException {
XContentBuilder xcb = IndexingUtils.createDoubleStringIndexMapping(
- XContentFactory.jsonBuilder(), "_default_");
+ XContentFactory.jsonBuilder(), "_default_");
+ //Added For Graphite Metrics
+ Timer.Context timePutIndex = mappingTimer.time();
PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices()
.preparePutTemplate("usergrid_template")
// set mapping as the default for all types
@@@ -590,30 -529,24 +604,30 @@@
String idString = IndexingUtils.idString(entityId).toLowerCase();
- final TermQueryBuilder tqb = QueryBuilders.termQuery( ENTITYID_ID_FIELDNAME, idString );
+ final TermQueryBuilder tqb = QueryBuilders.termQuery(ENTITYID_ID_FIELDNAME, idString);
+ //Added For Graphite Metrics
+ final Timer.Context timeDeleteAllVersions =allVersionsTimer.time();
final ListenableActionFuture<DeleteByQueryResponse> response = esProvider.getClient()
.prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
- response.addListener(new ActionListener<DeleteByQueryResponse>() {
+ response.addListener( new ActionListener<DeleteByQueryResponse>() {
+
@Override
- public void onResponse(DeleteByQueryResponse response) {
- logger.debug("Deleted entity {}:{} from all index scopes with response status = {}",
- entityId.getType(), entityId.getUuid(), response.status().toString());
+ public void onResponse( DeleteByQueryResponse response) {
+ timeDeleteAllVersions.stop();
+ logger
+ .debug("Deleted entity {}:{} from all index scopes with response status = {}", entityId.getType(),
+ entityId.getUuid(), response.status().toString() );
- checkDeleteByQueryResponse( tqb, response );
+ checkDeleteByQueryResponse(tqb, response);
}
+
@Override
- public void onFailure(Throwable e) {
- logger.error("Deleted entity {}:{} from all index scopes with error {}",
- entityId.getType(), entityId.getUuid(), e);
+ public void onFailure( Throwable e ) {
+ logger.error( "Deleted entity {}:{} from all index scopes with error {}", entityId.getType(),
+ entityId.getUuid(), e);
}
[30/31] incubator-usergrid git commit: Merge branch 'USERGRID-416'
into two-dot-o
Posted by sf...@apache.org.
Merge branch 'USERGRID-416' into two-dot-o
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/673c0573
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/673c0573
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/673c0573
Branch: refs/heads/USERGRID-405
Commit: 673c0573fa0ed0cccdae5141010cccc9a131163e
Parents: 710a1a3 8f67e92
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 9 09:13:39 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 9 09:13:39 2015 -0600
----------------------------------------------------------------------
.../Client/IUsergridClient.cs | 2 +-
.../Usergrid.Notifications/Client/PushClient.cs | 24 +++
.../Usergrid.Notifications/MainPage.xaml | 1 +
.../Usergrid.Notifications/MainPage.xaml.cs | 29 ++-
.../main/dist/init_instance/init_rest_server.sh | 3 -
.../src/main/groovy/configure_usergrid.groovy | 6 +-
stack/awscluster/ugcluster-cf.json | 8 +
stack/core/pom.xml | 7 +-
.../corepersistence/CpEntityManager.java | 109 +++++++++-
.../corepersistence/CpEntityManagerFactory.java | 156 +++++----------
.../corepersistence/CpRelationManager.java | 39 +++-
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++
.../corepersistence/results/EntityVerifier.java | 4 +-
.../results/FilteringLoader.java | 2 +-
.../cassandra/CassandraPersistenceUtils.java | 2 +-
.../main/resources/usergrid-core-context.xml | 14 +-
.../cassandra/EntityManagerFactoryImplIT.java | 1 +
.../core/metrics/MetricsFactoryImpl.java | 3 +-
.../persistence/index/EntityIndexFactory.java | 4 +-
.../persistence/index/IndexBufferProducer.java | 10 +-
.../usergrid/persistence/index/IndexFig.java | 12 +-
.../persistence/index/guice/IndexModule.java | 7 +-
.../index/impl/EsEntityIndexBatchImpl.java | 15 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 79 ++++++++
.../index/impl/EsEntityIndexImpl.java | 198 +++++++++++++++----
.../index/impl/EsIndexBufferConsumerImpl.java | 39 +++-
.../index/impl/EsIndexBufferProducerImpl.java | 30 ++-
.../persistence/index/impl/IndexingUtils.java | 5 +-
.../persistence/index/impl/EntityIndexTest.java | 2 +-
.../datagenerators/EntityDataGenerator.scala | 25 +++
.../datagenerators/FeederGenerator.scala | 18 +-
.../usergrid/scenarios/EntityScenarios.scala | 8 +-
.../org/apache/usergrid/settings/Settings.scala | 4 +-
.../simulations/PutCustomEntitySimulation.scala | 75 +++++++
stack/pom.xml | 24 ++-
stack/rest/pom.xml | 2 +-
.../org/apache/usergrid/rest/IndexResource.java | 19 ++
stack/services/pom.xml | 7 +-
.../services/notifications/wns/WNSAdapter.java | 39 ++++
.../resources/usergrid-services-context.xml | 8 +-
.../usergrid/management/OrganizationIT.java | 2 +-
42 files changed, 1050 insertions(+), 240 deletions(-)
----------------------------------------------------------------------
[29/31] incubator-usergrid git commit: Changed log level to warn to
catch issues with version mismatch in logging
Posted by sf...@apache.org.
Changed log level to warn to catch issues with version mismatch in logging
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8f67e92c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8f67e92c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8f67e92c
Branch: refs/heads/USERGRID-405
Commit: 8f67e92cff25f11fe759bfa2283537acca34788a
Parents: bcc5228
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Mar 9 00:46:07 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Mar 9 00:46:07 2015 -0600
----------------------------------------------------------------------
.../apache/usergrid/corepersistence/results/EntityVerifier.java | 4 ++--
.../apache/usergrid/corepersistence/results/FilteringLoader.java | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8f67e92c/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
index a35263c..075abc4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/EntityVerifier.java
@@ -85,7 +85,7 @@ public class EntityVerifier implements ResultsVerifier {
final UUID savedVersion = savedEntity.getVersion();
if ( UUIDComparator.staticCompare( savedVersion, candidateVersion ) > 0 ) {
- logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
entityId.getUuid(), entityId.getType(), candidateVersion, savedEntity
} );
@@ -96,7 +96,7 @@ public class EntityVerifier implements ResultsVerifier {
final Optional<org.apache.usergrid.persistence.model.entity.Entity> entity = savedEntity.getEntity();
if ( !entity.isPresent() ) {
- logger.warn( "Entity uuid:{} version v:{} is deleted but indexed, this is a bug ",
+ logger.warn( "Entity uuid:{} version v:{} is deleted but indexed, this is a bug ",
entityId.getUuid(), savedEntity.getEntity() );
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8f67e92c/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 7848be5..1afd76b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -147,7 +147,7 @@ public class FilteringLoader implements ResultsLoader {
if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
//de-index it
- logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
new Object[] {
entityId.getUuid(),
entityId.getType(),