You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/12 23:41:22 UTC
[15/38] incubator-usergrid git commit: Added caching to org/app lookup
Added caching to org/app lookup
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c8727c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c8727c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c8727c1
Branch: refs/heads/USERGRID-396
Commit: 4c8727c1dc0e5f02a498849be1a19e6342f6accf
Parents: 2c54a2b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 15:13:49 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 15:13:49 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 1 +
.../corepersistence/CpEntityManagerFactory.java | 116 ++++--------
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++++
.../main/resources/usergrid-core-context.xml | 14 +-
.../persistence/index/EntityIndexFactory.java | 4 +-
.../resources/usergrid-services-context.xml | 8 +-
7 files changed, 296 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5619382..652e084 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -676,6 +676,7 @@ public class CpEntityManager implements EntityManager {
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
+ Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager rmi = new CpRelationManager();
rmi.init( this, emf, applicationId, entityRef, null );
return rmi;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9aa36a0..16de53e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,6 +15,7 @@
*/
package org.apache.usergrid.corepersistence;
+import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -98,6 +99,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
});
+ private final OrgApplicationCache orgApplicationCache;
+
private ManagerCache managerCache;
private DataMigrationManager dataMigrationManager;
@@ -116,7 +119,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-
+ this.orgApplicationCache = new OrgApplicationCacheImpl( this );
}
@@ -156,18 +159,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
-// public ManagerCache getManagerCache() {
-//
-// if ( managerCache == null ) {
-// managerCache = injector.getInstance( ManagerCache.class );
-//
-// dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-// }
-// return managerCache;
-// }
-
-
-
@Override
public EntityManager getEntityManager(UUID applicationId) {
try {
@@ -207,15 +198,16 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
String appName = buildAppName( orgName, name );
- UUID applicationId = lookupApplication( appName );
- if ( applicationId != null ) {
+ final Optional<UUID> appId = orgApplicationCache.getApplicationId( appName );
+
+ if ( appId.isPresent() ) {
throw new ApplicationAlreadyExistsException( name );
}
- applicationId = UUIDGenerator.newTimeUUID();
+ UUID applicationId = UUIDGenerator.newTimeUUID();
- logger.debug( "New application orgName {} name {} id {} ",
+ logger.debug( "New application orgName {} orgAppName {} id {} ",
new Object[] { orgName, name, applicationId.toString() } );
initializeApplication( orgName, applicationId, appName, properties );
@@ -233,6 +225,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Map<String, Object> properties ) throws Exception {
+
+ //Ensure our management system exists before creating our application
+ init();
+
EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
final String appName = buildAppName( organizationName, name );
@@ -244,8 +240,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
getSetup().setupApplicationKeyspace( applicationId, appName );
- UUID orgUuid = lookupOrganization( organizationName );
- if ( orgUuid == null ) {
+
+ final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( name );
+
+
+ final UUID orgUuid;
+
+ if ( !cachedValue.isPresent() ) {
+
// create new org because the specified one does not exist
final String orgName = organizationName;
@@ -263,6 +265,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.refreshIndex();
orgUuid = orgInfo.getUuid();
+
+ //evit so it's re-loaded later
+ orgApplicationCache.evictOrgId( name );
+ } else{
+ orgUuid = cachedValue.get();
}
// create appinfo entry in the system app
@@ -293,6 +300,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
appEm.refreshIndex();
logger.info("Initialized application {}", appName );
+
+ //evict app Id from cache
+ orgApplicationCache.evictAppId( appName );
+
return applicationId;
}
@@ -374,71 +385,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public UUID lookupOrganization( String name ) throws Exception {
- init();
-
-
- // Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-
-
- final EntityRef alias = em.getAlias( "organizations", name );
-
- if ( alias == null ) {
- return null;
- }
-
- final Entity entity = em.get( alias );
-
- if ( entity == null ) {
- return null;
- }
-
- return entity.getUuid();
- // Results results = em.searchCollection( em.getApplicationRef(), "organizations", q );
- //
- // if ( results.isEmpty() ) {
- // return null;
- // }
- //
- // return results.iterator().next().getUuid();
- }
-
@Override
- public UUID lookupApplication( String name ) throws Exception {
- init();
-
- // TODO: why does this not work for restored apps
-
-// EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-// final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name );
-// if ( alias == null ) {
-// return null;
-// }
-// final Entity entity = em.get( alias );
-// if ( entity == null ) {
-// return null;
-// }
-// final UUID property = ( UUID ) entity.getProperty( "applicationUuid" );
-// return property;
-
- Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
-
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
-
- Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
-
- if ( results.isEmpty() ) {
- return null;
- }
-
- Entity entity = results.iterator().next();
- Object uuidObject = entity.getProperty("applicationUuid");
- if ( uuidObject instanceof UUID ) {
- return (UUID)uuidObject;
- }
- return UUIDUtils.tryExtractUUID( entity.getProperty("applicationUuid").toString() );
+ public UUID lookupApplication( String orgAppName ) throws Exception {
+ return orgApplicationCache.getApplicationId( orgAppName ).orNull();
}
@@ -632,7 +582,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.update( propsEntity );
} catch (Exception ex) {
- logger.error("Error deleting service property name: " + name, ex);
+ logger.error("Error deleting service property orgAppName: " + name, ex);
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
new file mode 100644
index 0000000..b20dbe1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Entity;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A simple cache interface for looking up entities from an EM
+ */
+public interface OrgApplicationCache {
+
+
+ /**
+ * Get an entity by it's alias property. The result is cached. To clear it call evict or evict all
+ * @param
+ * @return
+ */
+ public Optional<UUID> getOrganizationId(final String orgName);
+
+ /**
+ * Evict the org by name
+ * @param orgName
+ */
+ public void evictOrgId(final String orgName);
+
+ /**
+ * Evict the application by name
+ * @param applicationName
+ * @return
+ */
+ public Optional<UUID> getApplicationId(final String applicationName);
+
+
+ /**
+ * Evict the app id by the name
+ */
+ public void evictAppId(final String applicationname);
+
+
+ /**
+ * Evict all caches
+ */
+ public void evictAll();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
new file mode 100644
index 0000000..4baf598
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+
+
+/**
+ * Implements the org app cache for faster runtime lookups. These values are immutable, so this LRU cache can stay
+ * full for the duration of the execution
+ */
+public class OrgApplicationCacheImpl implements OrgApplicationCache {
+
+
+ /**
+ * Cache the pointer to our root entity manager for reference
+ */
+ private final EntityManager rootEm;
+
+ private final LoadingCache<String, Optional<UUID>> orgCache =
+ CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String, Optional<UUID>>() {
+ @Override
+ public Optional<UUID> load( final String key ) throws Exception {
+ return fetchOrganizationId( key );
+ }
+ } );
+
+
+ private final LoadingCache<String, Optional<UUID>> appCache =
+ CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String, Optional<UUID>>() {
+ @Override
+ public Optional<UUID> load( final String key ) throws Exception {
+ return fetchApplicationId( key );
+ }
+ } );
+
+
+ public OrgApplicationCacheImpl( final EntityManagerFactory emf ) {
+ this.rootEm = emf.getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
+ }
+
+
+ @Override
+ public Optional<UUID> getOrganizationId( final String orgName ) {
+ try {
+ return orgCache.get( orgName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to load org cache", e );
+ }
+ }
+
+
+ /**
+ * Fetches the organization
+ */
+ private Optional<UUID> fetchOrganizationId( final String orgName ) {
+
+ try {
+ final EntityRef alias = rootEm.getAlias( "organizations", orgName );
+
+ if ( alias == null ) {
+ return Optional.absent();
+ }
+
+ final Entity entity;
+
+ entity = rootEm.get( alias );
+
+
+ if ( entity == null ) {
+ return Optional.absent();
+ }
+
+ return Optional.of( entity.getUuid() );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load organization Id for caching", e );
+ }
+ }
+
+
+ @Override
+ public void evictOrgId( final String orgName ) {
+ orgCache.invalidate( orgName );
+ }
+
+
+ @Override
+ public Optional<UUID> getApplicationId( final String applicationName ) {
+ try {
+ return appCache.get( applicationName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to load org cache", e );
+ }
+ }
+
+
+ /**
+ * Fetch our application id
+ */
+ private Optional<UUID> fetchApplicationId( final String applicationName ) {
+
+ try {
+ Query q = Query.fromQL( PROPERTY_NAME + " = '" + applicationName + "'" );
+
+
+ Results results = rootEm.searchCollection( rootEm.getApplicationRef(), "appinfos", q );
+
+ if ( results.isEmpty() ) {
+ return Optional.absent();
+ }
+
+ Entity entity = results.iterator().next();
+ Object uuidObject = entity.getProperty( "applicationUuid" );
+
+ final UUID value;
+ if ( uuidObject instanceof UUID ) {
+ value = ( UUID ) uuidObject;
+ }
+ else {
+ value = UUIDUtils.tryExtractUUID( entity.getProperty( "applicationUuid" ).toString() );
+ }
+
+
+ return Optional.of( value );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to retreive application id", e );
+ }
+ }
+
+
+ @Override
+ public void evictAppId( final String applicationName ) {
+ appCache.invalidate( applicationName );
+ }
+
+
+ @Override
+ public void evictAll() {
+ orgCache.invalidateAll();
+ appCache.invalidateAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index c8ce0d8..cd40d6d 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -54,7 +54,7 @@
<constructor-arg value="${cassandra.url}" />
<!-- set the pool size if it's available. If not go with 50 -->
<property name="maxActive" value="${cassandra.connections:50}"/>
- <!--<property name="clockResolution" ref="microsecondsTimeResolution" />-->
+ <!--<property orgAppName="clockResolution" ref="microsecondsTimeResolution" />-->
<property name="opTimer" ref="taggedOpTimer"/>
<property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
</bean>
@@ -70,7 +70,7 @@
<bean id="loadBalancingPolicy" class="me.prettyprint.cassandra.connection.DynamicLoadBalancingPolicy"/>
<!-- locking for a single node -->
-<!-- <bean name="lockManager"
+<!-- <bean orgAppName="lockManager"
class="org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl" />-->
<!-- hector based locks -->
@@ -86,10 +86,10 @@
<!-- zookeeper locks -->
<!--
- <bean name="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl" >
- <property name="hostPort" value="${zookeeper.url}"/>
- <property name="sessionTimeout" value="2000"/>
- <property name="maxAttempts" value="10"/>
+ <bean orgAppName="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl" >
+ <property orgAppName="hostPort" value="${zookeeper.url}"/>
+ <property orgAppName="sessionTimeout" value="2000"/>
+ <property orgAppName="maxAttempts" value="10"/>
</bean> -->
@@ -196,7 +196,7 @@
</bean>
<bean id="jobFactory" class="org.apache.usergrid.batch.UsergridJobFactory" />
-
+
<context:component-scan base-package="org.apache.usergrid.batch.job" />
<context:annotation-config />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
index 78a5137..10752d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
@@ -25,6 +25,8 @@ import com.google.inject.assistedinject.Assisted;
public interface EntityIndexFactory {
- public EntityIndex createEntityIndex(
+ public EntityIndex createEntityIndex(
@Assisted ApplicationScope appScope);
+
+ void invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index 666051d..8674b71 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -103,10 +103,10 @@
<!--<bean id="importQueueListener" class="org.apache.usergrid.services.queues.ImportQueueListener"-->
<!--scope="singleton">-->
- <!--<constructor-arg name="emf" ref="entityManagerFactory" />-->
- <!--<constructor-arg name="metricsService" ref="metricsFactory" />-->
- <!--<constructor-arg name="smf" ref="serviceManagerFactory" />-->
- <!--<constructor-arg name="props" ref="properties" />-->
+ <!--<constructor-arg orgAppName="emf" ref="entityManagerFactory" />-->
+ <!--<constructor-arg orgAppName="metricsService" ref="metricsFactory" />-->
+ <!--<constructor-arg orgAppName="smf" ref="serviceManagerFactory" />-->
+ <!--<constructor-arg orgAppName="props" ref="properties" />-->
<!--</bean>-->