You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/29 16:26:58 UTC
[2/3] git commit: Add a short-term cache to avoid redundant Entity
reloading in CpEntityManager & CpRelationManager.
Add a short-term cache to avoid redundant Entity reloading in CpEntityManager & CpRelationManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c5719654
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c5719654
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c5719654
Branch: refs/heads/two-dot-o
Commit: c57196546f8c278b3eda3789015caa26d0c45229
Parents: 3d3f189
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Oct 29 10:38:14 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Oct 29 10:38:14 2014 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +
.../corepersistence/CpEntityManager.java | 73 +++++++++++++++++---
.../corepersistence/CpRelationManager.java | 31 +++++----
.../persistence/index/query/EntityResults.java | 6 +-
4 files changed, 87 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5719654/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index a4760dd..fb043d4 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -62,6 +62,9 @@ elasticsearch.port=9300
index.query.limit.default=1000
+usergrid.entity_cache_size=200
+usergrid.entity_cache_timeout_ms=500
+
###############################################################################
#
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5719654/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index c9ad87b..78c9433 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -17,6 +17,10 @@ package org.apache.usergrid.corepersistence;
import com.google.common.base.Preconditions;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.CacheLoader.InvalidCacheLoadException;
+import com.google.common.cache.LoadingCache;
import com.netflix.hystrix.exception.HystrixRuntimeException;
import com.yammer.metrics.annotation.Metered;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
@@ -37,6 +41,8 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;
import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import me.prettyprint.hector.api.Keyspace;
import me.prettyprint.hector.api.beans.ColumnSlice;
import me.prettyprint.hector.api.beans.CounterRow;
@@ -86,10 +92,8 @@ import static org.apache.usergrid.persistence.Schema.PROPERTY_TYPE;
import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
-import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
import org.apache.usergrid.persistence.SimpleEntityRef;
import static org.apache.usergrid.persistence.SimpleEntityRef.getUuid;
-import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
import org.apache.usergrid.persistence.SimpleRoleRef;
import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
@@ -131,7 +135,6 @@ import org.apache.usergrid.persistence.index.query.CounterResolution;
import org.apache.usergrid.persistence.index.query.Identifier;
import org.apache.usergrid.persistence.index.query.Query;
import org.apache.usergrid.persistence.index.query.Query.Level;
-import static org.apache.usergrid.persistence.index.query.Query.Level.REFS;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapScope;
import org.apache.usergrid.persistence.map.impl.MapScopeImpl;
@@ -140,7 +143,6 @@ import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.field.Field;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.apache.usergrid.utils.ClassUtils;
import static org.apache.usergrid.utils.ClassUtils.cast;
import org.apache.usergrid.utils.CompositeUtils;
@@ -185,6 +187,9 @@ public class CpEntityManager implements EntityManager {
private String TYPES_BY_UUID_MAP = "zzz_typesbyuuid_zzz";
+ /** Short-term cache to keep us from reloading same Entity during single request. */
+ private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
+
public CpEntityManager() {}
@@ -206,6 +211,56 @@ public class CpEntityManager implements EntityManager {
// set to false for now
this.skipAggregateCounters = false;
+
+ int entityCacheSize = Integer.parseInt(
+ cass.getProperties().getProperty("usergrid.entity_cache_size", "100"));
+
+ int entityCacheTimeout = Integer.parseInt(
+ cass.getProperties().getProperty("usergrid.entity_cache_timeout_ms", "500"));
+
+ this.entityCache = CacheBuilder.newBuilder()
+ .maximumSize( entityCacheSize )
+ .expireAfterWrite( entityCacheTimeout, TimeUnit.MILLISECONDS )
+ .build( new CacheLoader<EntityScope, org.apache.usergrid.persistence.model.entity.Entity>() {
+ public org.apache.usergrid.persistence.model.entity.Entity load( EntityScope entityScope ) {
+ return managerCache.getEntityCollectionManager(
+ entityScope.scope ).load( entityScope.entityId ).toBlocking().lastOrDefault(null);
+ }
+ }
+ );
+ }
+
+
+ /** Needed to support short-term Entity cache. */
+ public static class EntityScope {
+ CollectionScope scope;
+ Id entityId;
+ public EntityScope( CollectionScope scope, Id entityId ) {
+ this.scope = scope;
+ this.entityId = entityId;
+ }
+ }
+
+
+ /**
+ * Load entity from short-term cache.
+ * Package scope so that CpRelationManager can use it too.
+ *
+ * @param es Carries Entity Id and CollectionScope from which to load Entity.
+ * @return Entity or null if not found
+ */
+ org.apache.usergrid.persistence.model.entity.Entity load( EntityScope es ) {
+ try {
+ return entityCache.get( es );
+
+ } catch ( InvalidCacheLoadException icle ) {
+ // fine, entity not found
+ return null;
+
+ } catch ( ExecutionException exex ) {
+ // uh-oh, more serious problem
+ throw new RuntimeException( "Error loading entity", exex );
+ }
}
@@ -320,7 +375,7 @@ public class CpEntityManager implements EntityManager {
// }
org.apache.usergrid.persistence.model.entity.Entity cpEntity =
- ecm.load( id ).toBlockingObservable().last();
+ load( new EntityScope( collectionScope, id ) );
if ( cpEntity == null ) {
if ( logger.isDebugEnabled() ) {
@@ -407,7 +462,7 @@ public class CpEntityManager implements EntityManager {
// }
org.apache.usergrid.persistence.model.entity.Entity cpEntity =
- ecm.load( id ).toBlocking().last();
+ load( new EntityScope( collectionScope, id ) );
if ( cpEntity == null ) {
if ( logger.isDebugEnabled() ) {
@@ -497,6 +552,8 @@ public class CpEntityManager implements EntityManager {
try {
cpEntity = ecm.update( cpEntity ).toBlockingObservable().last();
+
+ // need to reload entity so bypass entity cache
cpEntity = ecm.load( entityId ).toBlockingObservable().last();
logger.debug("Wrote {}:{} version {}", new Object[] {
@@ -545,7 +602,7 @@ public class CpEntityManager implements EntityManager {
// }
org.apache.usergrid.persistence.model.entity.Entity entity =
- ecm.load( entityId ).toBlockingObservable().last();
+ load( new EntityScope( collectionScope, entityId ) );
if ( entity != null ) {
@@ -996,7 +1053,7 @@ public class CpEntityManager implements EntityManager {
// }
org.apache.usergrid.persistence.model.entity.Entity cpEntity =
- ecm.load( entityId ).toBlocking().last();
+ load( new EntityScope( collectionScope, entityId ) );
cpEntity.removeField( propertyName );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5719654/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index d0da90f..1e8dcc3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -234,9 +234,9 @@ public class CpRelationManager implements RelationManager {
} );
}
- //TODO PERFORMANCE why are we loading this again here?
- this.cpHeadEntity = ecm.load( new SimpleId(
- headEntity.getUuid(), headEntity.getType() )).toBlocking().lastOrDefault(null);
+ Id entityId = new SimpleId( headEntity.getUuid(), headEntity.getType() );
+ this.cpHeadEntity = ((CpEntityManager)em).load(
+ new CpEntityManager.EntityScope( headEntityScope, entityId));
// commented out because it is possible that CP entity has not been created yet
Assert.notNull( cpHeadEntity, "cpHeadEntity cannot be null" );
@@ -626,12 +626,12 @@ public class CpRelationManager implements RelationManager {
applicationScope.getApplication(),
applicationScope.getApplication(),
CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
-
EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager( memberScope );
//TODO, this double load should disappear once events are in
- org.apache.usergrid.persistence.model.entity.Entity memberEntity = memberMgr.load(
- new SimpleId( itemRef.getUuid(), itemRef.getType() ) ).toBlocking().last();
+ Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
+ org.apache.usergrid.persistence.model.entity.Entity memberEntity =
+ ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId));
if ( memberEntity == null ) {
throw new RuntimeException(
@@ -796,8 +796,9 @@ public class CpRelationManager implements RelationManager {
});
}
- org.apache.usergrid.persistence.model.entity.Entity memberEntity = memberMgr.load(
- new SimpleId( itemRef.getUuid(), itemRef.getType() ) ).toBlockingObservable().last();
+ Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
+ org.apache.usergrid.persistence.model.entity.Entity memberEntity =
+ ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId));
final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
final EntityIndexBatch batch = ei.createBatch();
@@ -1015,9 +1016,9 @@ public class CpRelationManager implements RelationManager {
});
}
- org.apache.usergrid.persistence.model.entity.Entity targetEntity = targetEcm.load(
- new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() ) )
- .toBlockingObservable().last();
+ Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType());
+ org.apache.usergrid.persistence.model.entity.Entity targetEntity =
+ ((CpEntityManager)em).load( new CpEntityManager.EntityScope( targetScope, entityId));
String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType );
@@ -1240,12 +1241,12 @@ public class CpRelationManager implements RelationManager {
connectingEntityRef.getUuid(),
connectedEntityRef.getType(),
connectedEntityRef.getUuid()
- } );
+ });
}
- org.apache.usergrid.persistence.model.entity.Entity targetEntity = targetEcm.load(
- new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() ) )
- .toBlockingObservable().last();
+ Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() );
+ org.apache.usergrid.persistence.model.entity.Entity targetEntity =
+ ((CpEntityManager)em).load( new CpEntityManager.EntityScope( targetScope, entityId));
// Delete graph edge connection from head entity to member entity
Edge edge = new SimpleEdge(
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c5719654/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/EntityResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/EntityResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/EntityResults.java
index 4ce3848..b2aef93 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/EntityResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/EntityResults.java
@@ -67,18 +67,18 @@ public class EntityResults implements Iterable<Entity>, Iterator<Entity> {
/**
- * Advance to our next candidate so that it is avaiablel
+ * Advance to our next candidate so that it is available
*/
private void doAdvance(){
while(itr.hasNext() && next == null){
CandidateResult candidate = itr.next();
- //our candidate is > our max, we can't use it
+ // our candidate is > our max, we can't use it
if( UUIDUtils.compare( candidate.getVersion(), maxVersion ) > 0){
continue;
}
- //our candidate was too new, ignore it
+ // our candidate was too new, ignore it
next = ecm.load( candidate.getId() ).toBlocking().single();
}
}