You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/03/07 00:10:35 UTC
[1/4] incubator-usergrid git commit: Updated tomcat threads to be
configurable per core in cloud formation template
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-416 [created] ecc1ddeed
Updated tomcat threads to be configurable per core in cloud formation template
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/10ec6354
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/10ec6354
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/10ec6354
Branch: refs/heads/USERGRID-416
Commit: 10ec6354c9d6128b28811ba969d2ba3eec24b624
Parents: 710a1a3
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 13:35:17 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 13:35:17 2015 -0700
----------------------------------------------------------------------
.../src/main/dist/init_instance/init_rest_server.sh | 3 ---
stack/awscluster/src/main/groovy/configure_usergrid.groovy | 6 +++---
stack/awscluster/ugcluster-cf.json | 8 ++++++++
3 files changed, 11 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ec6354/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 3860cd4..cad4e74 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -68,9 +68,6 @@ export ACCEPT_COUNT=100
export NR_OPEN=1048576
export FILE_MAX=761773
-#Number of threads to allow per core
-export NUM_THREAD_PROC=25
-
#Get the number of processors
export NUM_PROC=$(nproc)
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ec6354/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index 619c0b4..1f7140c 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -50,9 +50,9 @@ def esShards = numEsNodes;
//This gives us 3 copies, which means we'll have a quorum with primary + 1 replica
def esReplicas = 1;
-def cassThreads = System.getenv().get("TOMCAT_THREADS")
+def tomcatThreads = System.getenv().get("TOMCAT_THREADS")
//temporarily set to equal since we now have a sane tomcat thread calculation
-def hystrixThreads = cassThreads
+def hystrixThreads = tomcatThreads
//if we end in -1, we remove it
def ec2Region = System.getenv().get("EC2_REGION")
@@ -99,7 +99,7 @@ cassandra.keyspace.strategy=org.apache.cassandra.locator.NetworkTopologyStrategy
cassandra.keyspace.replication=${cassEc2Region}:${replFactor}
cassandra.timeout=5000
-cassandra.connections=${cassThreads}
+cassandra.connections=${tomcatThreads}
hystrix.threadpool.graph_user.coreSize=${hystrixThreads}
hystrix.threadpool.graph_async.coreSize=${hystrixThreads}
usergrid.read.cl=${readConsistencyLevel}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/10ec6354/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 9bce545..7926929 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -46,6 +46,12 @@
],
"ConstraintDescription": "must be valid instance type."
},
+ "TomcatThreadsPerCore": {
+ "Description": "Number of threads to configure tomcat for per core",
+ "Type": "Number",
+ "Default": "50",
+ "MinValue": "1"
+ },
"KeyPair": {
"Description": "Name of an existing EC2 KeyPair to enable SSH access to the instance",
"Type": "AWS::EC2::KeyPair::KeyName",
@@ -1171,6 +1177,8 @@
"\n",
"export RELEASE_BUCKET=", { "Ref":"ReleaseBucket" }, "\n",
"\n",
+ "export NUM_THREAD_PROC=", { "Ref":"TomcatThreadsPerCore" }, "\n",
+ "\n",
"export SUPER_USER_EMAIL=", { "Ref":"SuperUserEmail" }, "\n",
"export TEST_ADMIN_USER_EMAIL=", { "Ref":"TestAdminUserEmail" }, "\n",
"\n",
[3/4] incubator-usergrid git commit: Added caching to org/app lookup
Posted by to...@apache.org.
Added caching to org/app lookup
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/4c8727c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/4c8727c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/4c8727c1
Branch: refs/heads/USERGRID-416
Commit: 4c8727c1dc0e5f02a498849be1a19e6342f6accf
Parents: 2c54a2b
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 15:13:49 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 15:13:49 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 1 +
.../corepersistence/CpEntityManagerFactory.java | 116 ++++--------
.../corepersistence/OrgApplicationCache.java | 67 +++++++
.../OrgApplicationCacheImpl.java | 181 +++++++++++++++++++
.../main/resources/usergrid-core-context.xml | 14 +-
.../persistence/index/EntityIndexFactory.java | 4 +-
.../resources/usergrid-services-context.xml | 8 +-
7 files changed, 296 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 5619382..652e084 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -676,6 +676,7 @@ public class CpEntityManager implements EntityManager {
@Override
public RelationManager getRelationManager( EntityRef entityRef ) {
+ Preconditions.checkNotNull( entityRef, "entityRef cannot be null" );
CpRelationManager rmi = new CpRelationManager();
rmi.init( this, emf, applicationId, entityRef, null );
return rmi;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 9aa36a0..16de53e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -15,6 +15,7 @@
*/
package org.apache.usergrid.corepersistence;
+import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -98,6 +99,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
});
+ private final OrgApplicationCache orgApplicationCache;
+
private ManagerCache managerCache;
private DataMigrationManager dataMigrationManager;
@@ -116,7 +119,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.managerCache = injector.getInstance( ManagerCache.class );
this.dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-
+ this.orgApplicationCache = new OrgApplicationCacheImpl( this );
}
@@ -156,18 +159,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
-// public ManagerCache getManagerCache() {
-//
-// if ( managerCache == null ) {
-// managerCache = injector.getInstance( ManagerCache.class );
-//
-// dataMigrationManager = injector.getInstance( DataMigrationManager.class );
-// }
-// return managerCache;
-// }
-
-
-
@Override
public EntityManager getEntityManager(UUID applicationId) {
try {
@@ -207,15 +198,16 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
String appName = buildAppName( orgName, name );
- UUID applicationId = lookupApplication( appName );
- if ( applicationId != null ) {
+ final Optional<UUID> appId = orgApplicationCache.getApplicationId( appName );
+
+ if ( appId.isPresent() ) {
throw new ApplicationAlreadyExistsException( name );
}
- applicationId = UUIDGenerator.newTimeUUID();
+ UUID applicationId = UUIDGenerator.newTimeUUID();
- logger.debug( "New application orgName {} name {} id {} ",
+ logger.debug( "New application orgName {} orgAppName {} id {} ",
new Object[] { orgName, name, applicationId.toString() } );
initializeApplication( orgName, applicationId, appName, properties );
@@ -233,6 +225,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
Map<String, Object> properties ) throws Exception {
+
+ //Ensure our management system exists before creating our application
+ init();
+
EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID);
final String appName = buildAppName( organizationName, name );
@@ -244,8 +240,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
getSetup().setupApplicationKeyspace( applicationId, appName );
- UUID orgUuid = lookupOrganization( organizationName );
- if ( orgUuid == null ) {
+
+ final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( name );
+
+
+ final UUID orgUuid;
+
+ if ( !cachedValue.isPresent() ) {
+
// create new org because the specified one does not exist
final String orgName = organizationName;
@@ -263,6 +265,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.refreshIndex();
orgUuid = orgInfo.getUuid();
+
+ //evit so it's re-loaded later
+ orgApplicationCache.evictOrgId( name );
+ } else{
+ orgUuid = cachedValue.get();
}
// create appinfo entry in the system app
@@ -293,6 +300,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
appEm.refreshIndex();
logger.info("Initialized application {}", appName );
+
+ //evict app Id from cache
+ orgApplicationCache.evictAppId( appName );
+
return applicationId;
}
@@ -374,71 +385,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
- public UUID lookupOrganization( String name ) throws Exception {
- init();
-
-
- // Query q = Query.fromQL(PROPERTY_NAME + " = '" + name + "'");
- EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-
-
- final EntityRef alias = em.getAlias( "organizations", name );
-
- if ( alias == null ) {
- return null;
- }
-
- final Entity entity = em.get( alias );
-
- if ( entity == null ) {
- return null;
- }
-
- return entity.getUuid();
- // Results results = em.searchCollection( em.getApplicationRef(), "organizations", q );
- //
- // if ( results.isEmpty() ) {
- // return null;
- // }
- //
- // return results.iterator().next().getUuid();
- }
-
@Override
- public UUID lookupApplication( String name ) throws Exception {
- init();
-
- // TODO: why does this not work for restored apps
-
-// EntityManager em = getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
-// final EntityRef alias = em.getAlias( CpNamingUtils.APPINFOS, name );
-// if ( alias == null ) {
-// return null;
-// }
-// final Entity entity = em.get( alias );
-// if ( entity == null ) {
-// return null;
-// }
-// final UUID property = ( UUID ) entity.getProperty( "applicationUuid" );
-// return property;
-
- Query q = Query.fromQL( PROPERTY_NAME + " = '" + name + "'");
-
- EntityManager em = getEntityManager(CpNamingUtils.SYSTEM_APP_ID);
-
- Results results = em.searchCollection( em.getApplicationRef(), "appinfos", q);
-
- if ( results.isEmpty() ) {
- return null;
- }
-
- Entity entity = results.iterator().next();
- Object uuidObject = entity.getProperty("applicationUuid");
- if ( uuidObject instanceof UUID ) {
- return (UUID)uuidObject;
- }
- return UUIDUtils.tryExtractUUID( entity.getProperty("applicationUuid").toString() );
+ public UUID lookupApplication( String orgAppName ) throws Exception {
+ return orgApplicationCache.getApplicationId( orgAppName ).orNull();
}
@@ -632,7 +582,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.update( propsEntity );
} catch (Exception ex) {
- logger.error("Error deleting service property name: " + name, ex);
+ logger.error("Error deleting service property orgAppName: " + name, ex);
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
new file mode 100644
index 0000000..b20dbe1
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCache.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+
+import org.apache.usergrid.persistence.Entity;
+
+import com.google.common.base.Optional;
+
+
+/**
+ * A simple cache interface for looking up entities from an EM
+ */
+public interface OrgApplicationCache {
+
+
+ /**
+ * Get an entity by it's alias property. The result is cached. To clear it call evict or evict all
+ * @param
+ * @return
+ */
+ public Optional<UUID> getOrganizationId(final String orgName);
+
+ /**
+ * Evict the org by name
+ * @param orgName
+ */
+ public void evictOrgId(final String orgName);
+
+ /**
+ * Evict the application by name
+ * @param applicationName
+ * @return
+ */
+ public Optional<UUID> getApplicationId(final String applicationName);
+
+
+ /**
+ * Evict the app id by the name
+ */
+ public void evictAppId(final String applicationname);
+
+
+ /**
+ * Evict all caches
+ */
+ public void evictAll();
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
new file mode 100644
index 0000000..4baf598
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/OrgApplicationCacheImpl.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.usergrid.corepersistence;
+
+
+import java.util.UUID;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.utils.UUIDUtils;
+
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+
+
+/**
+ * Implements the org app cache for faster runtime lookups. These values are immutable, so this LRU cache can stay
+ * full for the duration of the execution
+ */
+public class OrgApplicationCacheImpl implements OrgApplicationCache {
+
+
+ /**
+ * Cache the pointer to our root entity manager for reference
+ */
+ private final EntityManager rootEm;
+
+ private final LoadingCache<String, Optional<UUID>> orgCache =
+ CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String, Optional<UUID>>() {
+ @Override
+ public Optional<UUID> load( final String key ) throws Exception {
+ return fetchOrganizationId( key );
+ }
+ } );
+
+
+ private final LoadingCache<String, Optional<UUID>> appCache =
+ CacheBuilder.newBuilder().maximumSize( 10000 ).build( new CacheLoader<String, Optional<UUID>>() {
+ @Override
+ public Optional<UUID> load( final String key ) throws Exception {
+ return fetchApplicationId( key );
+ }
+ } );
+
+
+ public OrgApplicationCacheImpl( final EntityManagerFactory emf ) {
+ this.rootEm = emf.getEntityManager( CpNamingUtils.SYSTEM_APP_ID );
+ }
+
+
+ @Override
+ public Optional<UUID> getOrganizationId( final String orgName ) {
+ try {
+ return orgCache.get( orgName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to load org cache", e );
+ }
+ }
+
+
+ /**
+ * Fetches the organization
+ */
+ private Optional<UUID> fetchOrganizationId( final String orgName ) {
+
+ try {
+ final EntityRef alias = rootEm.getAlias( "organizations", orgName );
+
+ if ( alias == null ) {
+ return Optional.absent();
+ }
+
+ final Entity entity;
+
+ entity = rootEm.get( alias );
+
+
+ if ( entity == null ) {
+ return Optional.absent();
+ }
+
+ return Optional.of( entity.getUuid() );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to load organization Id for caching", e );
+ }
+ }
+
+
+ @Override
+ public void evictOrgId( final String orgName ) {
+ orgCache.invalidate( orgName );
+ }
+
+
+ @Override
+ public Optional<UUID> getApplicationId( final String applicationName ) {
+ try {
+ return appCache.get( applicationName );
+ }
+ catch ( ExecutionException e ) {
+ throw new RuntimeException( "Unable to load org cache", e );
+ }
+ }
+
+
+ /**
+ * Fetch our application id
+ */
+ private Optional<UUID> fetchApplicationId( final String applicationName ) {
+
+ try {
+ Query q = Query.fromQL( PROPERTY_NAME + " = '" + applicationName + "'" );
+
+
+ Results results = rootEm.searchCollection( rootEm.getApplicationRef(), "appinfos", q );
+
+ if ( results.isEmpty() ) {
+ return Optional.absent();
+ }
+
+ Entity entity = results.iterator().next();
+ Object uuidObject = entity.getProperty( "applicationUuid" );
+
+ final UUID value;
+ if ( uuidObject instanceof UUID ) {
+ value = ( UUID ) uuidObject;
+ }
+ else {
+ value = UUIDUtils.tryExtractUUID( entity.getProperty( "applicationUuid" ).toString() );
+ }
+
+
+ return Optional.of( value );
+ }
+ catch ( Exception e ) {
+ throw new RuntimeException( "Unable to retreive application id", e );
+ }
+ }
+
+
+ @Override
+ public void evictAppId( final String applicationName ) {
+ appCache.invalidate( applicationName );
+ }
+
+
+ @Override
+ public void evictAll() {
+ orgCache.invalidateAll();
+ appCache.invalidateAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/core/src/main/resources/usergrid-core-context.xml
----------------------------------------------------------------------
diff --git a/stack/core/src/main/resources/usergrid-core-context.xml b/stack/core/src/main/resources/usergrid-core-context.xml
index c8ce0d8..cd40d6d 100644
--- a/stack/core/src/main/resources/usergrid-core-context.xml
+++ b/stack/core/src/main/resources/usergrid-core-context.xml
@@ -54,7 +54,7 @@
<constructor-arg value="${cassandra.url}" />
<!-- set the pool size if it's available. If not go with 50 -->
<property name="maxActive" value="${cassandra.connections:50}"/>
- <!--<property name="clockResolution" ref="microsecondsTimeResolution" />-->
+ <!--<property orgAppName="clockResolution" ref="microsecondsTimeResolution" />-->
<property name="opTimer" ref="taggedOpTimer"/>
<property name="loadBalancingPolicy" ref="loadBalancingPolicy"/>
</bean>
@@ -70,7 +70,7 @@
<bean id="loadBalancingPolicy" class="me.prettyprint.cassandra.connection.DynamicLoadBalancingPolicy"/>
<!-- locking for a single node -->
-<!-- <bean name="lockManager"
+<!-- <bean orgAppName="lockManager"
class="org.apache.usergrid.locking.singlenode.SingleNodeLockManagerImpl" />-->
<!-- hector based locks -->
@@ -86,10 +86,10 @@
<!-- zookeeper locks -->
<!--
- <bean name="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl" >
- <property name="hostPort" value="${zookeeper.url}"/>
- <property name="sessionTimeout" value="2000"/>
- <property name="maxAttempts" value="10"/>
+ <bean orgAppName="lockManager" class="org.apache.usergrid.locking.zookeeper.ZooKeeperLockManagerImpl" >
+ <property orgAppName="hostPort" value="${zookeeper.url}"/>
+ <property orgAppName="sessionTimeout" value="2000"/>
+ <property orgAppName="maxAttempts" value="10"/>
</bean> -->
@@ -196,7 +196,7 @@
</bean>
<bean id="jobFactory" class="org.apache.usergrid.batch.UsergridJobFactory" />
-
+
<context:component-scan base-package="org.apache.usergrid.batch.job" />
<context:annotation-config />
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
index 78a5137..10752d1 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexFactory.java
@@ -25,6 +25,8 @@ import com.google.inject.assistedinject.Assisted;
public interface EntityIndexFactory {
- public EntityIndex createEntityIndex(
+ public EntityIndex createEntityIndex(
@Assisted ApplicationScope appScope);
+
+ void invalidate();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/4c8727c1/stack/services/src/main/resources/usergrid-services-context.xml
----------------------------------------------------------------------
diff --git a/stack/services/src/main/resources/usergrid-services-context.xml b/stack/services/src/main/resources/usergrid-services-context.xml
index 666051d..8674b71 100644
--- a/stack/services/src/main/resources/usergrid-services-context.xml
+++ b/stack/services/src/main/resources/usergrid-services-context.xml
@@ -103,10 +103,10 @@
<!--<bean id="importQueueListener" class="org.apache.usergrid.services.queues.ImportQueueListener"-->
<!--scope="singleton">-->
- <!--<constructor-arg name="emf" ref="entityManagerFactory" />-->
- <!--<constructor-arg name="metricsService" ref="metricsFactory" />-->
- <!--<constructor-arg name="smf" ref="serviceManagerFactory" />-->
- <!--<constructor-arg name="props" ref="properties" />-->
+ <!--<constructor-arg orgAppName="emf" ref="entityManagerFactory" />-->
+ <!--<constructor-arg orgAppName="metricsService" ref="metricsFactory" />-->
+ <!--<constructor-arg orgAppName="smf" ref="serviceManagerFactory" />-->
+ <!--<constructor-arg orgAppName="props" ref="properties" />-->
<!--</bean>-->
[2/4] incubator-usergrid git commit: Upgraded plexus utils and fixes
build concurrency issues with surefire
Posted by to...@apache.org.
Upgraded plexus utils and fixes build concurrency issues with surefire
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2c54a2b6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2c54a2b6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2c54a2b6
Branch: refs/heads/USERGRID-416
Commit: 2c54a2b6fa4314f696f78f2b2cf11ebcaac03d1e
Parents: 10ec635
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 4 15:57:11 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 15:11:22 2015 -0700
----------------------------------------------------------------------
stack/core/pom.xml | 7 +-
.../core/metrics/MetricsFactoryImpl.java | 3 +-
.../persistence/index/guice/IndexModule.java | 7 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 74 ++++++++++++++++++++
stack/pom.xml | 24 ++++---
stack/rest/pom.xml | 2 +-
stack/services/pom.xml | 7 +-
.../usergrid/management/OrganizationIT.java | 2 +-
8 files changed, 109 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/core/pom.xml
----------------------------------------------------------------------
diff --git a/stack/core/pom.xml b/stack/core/pom.xml
index 77ce82e..d30bedb 100644
--- a/stack/core/pom.xml
+++ b/stack/core/pom.xml
@@ -101,9 +101,14 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
+ </dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
index 2528070..773828d 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/metrics/MetricsFactoryImpl.java
@@ -35,7 +35,7 @@ import java.util.concurrent.TimeUnit;
@Singleton
public class MetricsFactoryImpl implements MetricsFactory {
- private final MetricsFig metricsFig;
+
private MetricRegistry registry;
private GraphiteReporter graphiteReporter;
private JmxReporter jmxReporter;
@@ -44,7 +44,6 @@ public class MetricsFactoryImpl implements MetricsFactory {
@Inject
public MetricsFactoryImpl(MetricsFig metricsFig) {
- this.metricsFig = metricsFig;
registry = new MetricRegistry();
String metricsHost = metricsFig.getHost();
if(!metricsHost.equals("false")) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
index d9a14c9..6fee17e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/guice/IndexModule.java
@@ -22,6 +22,8 @@ package org.apache.usergrid.persistence.index.guice;
import org.apache.usergrid.persistence.index.*;
import com.google.inject.AbstractModule;
import com.google.inject.assistedinject.FactoryModuleBuilder;
+
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexFactoryImpl;
import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferConsumerImpl;
import org.apache.usergrid.persistence.index.impl.EsIndexBufferProducerImpl;
@@ -36,9 +38,8 @@ public class IndexModule extends AbstractModule {
// install our configuration
install(new GuicyFigModule(IndexFig.class));
- install(new FactoryModuleBuilder()
- .implement(EntityIndex.class, EsEntityIndexImpl.class)
- .build(EntityIndexFactory.class));
+
+ bind(EntityIndexFactory.class).to( EsEntityIndexFactoryImpl.class );
bind(IndexBufferProducer.class).to(EsIndexBufferProducerImpl.class);
bind(IndexBufferConsumer.class).to(EsIndexBufferConsumerImpl.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
new file mode 100644
index 0000000..4bf8b59
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -0,0 +1,74 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
+import com.google.inject.assistedinject.Assisted;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexBufferProducer;
+import org.apache.usergrid.persistence.index.IndexFig;
+
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Get index from factory, adds caching
+ */
+public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
+
+ private final IndexFig config;
+ private final EsProvider provider;
+ private final EsIndexCache indexCache;
+ private final IndexBufferProducer indexBatchBufferProducer;
+
+ private LoadingCache<ApplicationScope, EntityIndex> eiCache =
+ CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, EntityIndex>() {
+ public EntityIndex load( ApplicationScope scope ) {
+ return new EsEntityIndexImpl(scope,config, indexBatchBufferProducer, provider,indexCache);
+ }
+ } );
+
+ @Inject
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final EsIndexCache indexCache,
+ final IndexBufferProducer indexBatchBufferProducer ){
+ this.config = config;
+ this.provider = provider;
+ this.indexCache = indexCache;
+ this.indexBatchBufferProducer = indexBatchBufferProducer;
+ }
+
+ @Override
+ public EntityIndex createEntityIndex(final ApplicationScope appScope) {
+ try{
+ return eiCache.get(appScope);
+ }catch (ExecutionException ee){
+ throw new RuntimeException(ee);
+ }
+ }
+
+ @Override
+ public void invalidate() {
+ eiCache.invalidateAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index 53755b3..25b2258 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -127,6 +127,7 @@
<metrics.version>3.0.0</metrics.version>
<rx.version>0.19.6</rx.version>
+ <surefire.plugin.artifactName>surefire-junit47</surefire.plugin.artifactName>
<surefire.plugin.version>2.18.1</surefire.plugin.version>
<powermock.version>1.6.1</powermock.version>
@@ -1503,8 +1504,8 @@
<forkCount>${usergrid.it.forkCount}</forkCount>
<reuseForks>${usergrid.it.reuseForks}</reuseForks>
<threadCount>${usergrid.it.forkCount}</threadCount>
- <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}
- </argLine>
+ <argLine>-Xmx${ug.heapmax} -Xms${ug.heapmin} -javaagent:${settings.localRepository}/com/github/stephenc/jamm/0.2.5/jamm-0.2.5.jar ${ug.argline}</argLine>
+ <testFailureIgnore>false</testFailureIgnore>
</configuration>
<!-- TODO, we may need an exclusion. Appears to be a classloader bug
@@ -1515,12 +1516,19 @@
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>${surefire.plugin.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.maven.surfire</groupId>
- <artifactId>common-junit3</artifactId>
- </exclusion>
- </exclusions>
+
+ <!--<exclusions>-->
+ <!--<exclusion>-->
+ <!--<groupId>org.apache.maven.surfire</groupId>-->
+ <!--<artifactId>common-junit3</artifactId>-->
+ <!--</exclusion>-->
+ <!--</exclusions>-->
+ </dependency>
+ <!-- override plex utils, otherwise bug from above SO post happens-->
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
</dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/rest/pom.xml
----------------------------------------------------------------------
diff --git a/stack/rest/pom.xml b/stack/rest/pom.xml
index 3ad58b4..5a856de 100644
--- a/stack/rest/pom.xml
+++ b/stack/rest/pom.xml
@@ -115,7 +115,7 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/services/pom.xml
----------------------------------------------------------------------
diff --git a/stack/services/pom.xml b/stack/services/pom.xml
index 46b0878..f5e8dd5 100644
--- a/stack/services/pom.xml
+++ b/stack/services/pom.xml
@@ -108,9 +108,14 @@
<dependencies>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
- <artifactId>surefire-junit47</artifactId>
+ <artifactId>${surefire.plugin.artifactName}</artifactId>
<version>${surefire.plugin.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.codehaus.plexus</groupId>
+ <artifactId>plexus-utils</artifactId>
+ <version>3.0.21</version>
+ </dependency>
</dependencies>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2c54a2b6/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
index bd96434..5bf0c3d 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/OrganizationIT.java
@@ -90,7 +90,7 @@ public class OrganizationIT {
OrganizationInfo organization2 = setup.getMgmtSvc().getOrganizationForApplication( applicationId );
assertNotNull( organization2 );
- assertEquals( "wrong organization name", orgName, organization2.getName() );
+ assertEquals( "wrong organization name", organization.getOrganization().getName(), organization2.getName() );
boolean verified = setup.getMgmtSvc().verifyAdminUserPassword( organization.getOwner().getUuid(), "test" );
assertTrue( verified );
[4/4] incubator-usergrid git commit: Fixes lookup cache bug
Posted by to...@apache.org.
Fixes lookup cache bug
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ecc1ddee
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ecc1ddee
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ecc1ddee
Branch: refs/heads/USERGRID-416
Commit: ecc1ddeed2f2e54a66c2b5a56eed689593380820
Parents: 4c8727c
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Mar 6 16:10:33 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Mar 6 16:10:33 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 31 ++++++++++----------
1 file changed, 16 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ecc1ddee/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 16de53e..e0029c1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -241,10 +241,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
getSetup().setupApplicationKeyspace( applicationId, appName );
- final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( name );
+ final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
- final UUID orgUuid;
+ UUID orgUuid;
if ( !cachedValue.isPresent() ) {
@@ -252,22 +252,22 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// create new org because the specified one does not exist
final String orgName = organizationName;
- final Entity orgInfo;
+
try {
- orgInfo = em.create( "organization", new HashMap<String, Object>() {{
+ final Entity orgInfo = em.create( "organization", new HashMap<String, Object>() {{
put( PROPERTY_NAME, orgName );
}} );
+ orgUuid = orgInfo.getUuid();
+ //evit so it's re-loaded later
+ orgApplicationCache.evictOrgId( name );
}
catch ( DuplicateUniquePropertyExistsException e ) {
- throw new OrganizationAlreadyExistsException( orgName );
+ //swallow, if it exists, just get it
+ orgApplicationCache.evictOrgId( organizationName );
+ orgUuid = orgApplicationCache.getOrganizationId( organizationName ).get();
}
- em.refreshIndex();
- orgUuid = orgInfo.getUuid();
-
- //evit so it's re-loaded later
- orgApplicationCache.evictOrgId( name );
} else{
orgUuid = cachedValue.get();
}
@@ -281,11 +281,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
put( "organizationUuid", orgId );
}};
- try{
- em.create( "appinfo", appInfoMap );
- }catch(DuplicateUniquePropertyExistsException e){
- throw new ApplicationAlreadyExistsException( appName );
- }
+ try {
+ em.create( "appinfo", appInfoMap );
+ }
+ catch ( DuplicateUniquePropertyExistsException e ) {
+ throw new ApplicationAlreadyExistsException( appName );
+ }
em.refreshIndex();
// create application entity