You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/06/03 14:45:51 UTC
[01/26] usergrid git commit: Move the initial get of management app
to the CpEntityManagerFactory with retries,
and caching for the management app itself.
Repository: usergrid
Updated Branches:
refs/heads/release-2.1.1 0970e1dfa -> a1cb1f5f4
Move the initial get of management app to the CpEntityManagerFactory with retries, and caching for the management app itself.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e0c0c875
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e0c0c875
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e0c0c875
Branch: refs/heads/release-2.1.1
Commit: e0c0c875271cda47f9baf9072f029a92921fd1be
Parents: 7fdca3d
Author: Dave Johnson <sn...@apache.org>
Authored: Mon May 16 16:10:07 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon May 16 16:10:07 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 52 ++++-----
.../corepersistence/CpEntityManagerFactory.java | 112 ++++++++++++++-----
.../usergrid/services/ServiceManager.java | 59 ++--------
3 files changed, 113 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0c0c875/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 68f5d71..c8f3253 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -36,6 +36,7 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,24 +49,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
-import org.apache.usergrid.persistence.AggregateCounter;
-import org.apache.usergrid.persistence.AggregateCounterSet;
-import org.apache.usergrid.persistence.CollectionRef;
-import org.apache.usergrid.persistence.ConnectedEntityRef;
-import org.apache.usergrid.persistence.ConnectionRef;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.IndexBucketLocator;
-import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.Level;
-import org.apache.usergrid.persistence.RelationManager;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.Schema;
-import org.apache.usergrid.persistence.SimpleEntityRef;
-import org.apache.usergrid.persistence.SimpleRoleRef;
-import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -230,6 +214,9 @@ public class CpEntityManager implements EntityManager {
private EntityCollectionManager ecm;
+ private CpEntityManagerFactory emf;
+
+
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -242,18 +229,20 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager( final CassandraService cass,
- final CounterUtils counterUtils,
- final AsyncEventService indexService,
- final ManagerCache managerCache,
- final MetricsFactory metricsFactory,
- final EntityManagerFig entityManagerFig,
- final GraphManagerFactory graphManagerFactory,
- final CollectionService collectionService,
- final ConnectionService connectionService,
- final IndexSchemaCacheFactory indexSchemaCacheFactory,
- final UUID applicationId ) {
-
+ public CpEntityManager(final CpEntityManagerFactory emf,
+ final CassandraService cass,
+ final CounterUtils counterUtils,
+ final AsyncEventService indexService,
+ final ManagerCache managerCache,
+ final MetricsFactory metricsFactory,
+ final EntityManagerFig entityManagerFig,
+ final GraphManagerFactory graphManagerFactory,
+ final CollectionService collectionService,
+ final ConnectionService connectionService,
+ final IndexSchemaCacheFactory indexSchemaCacheFactory,
+ final UUID applicationId ) {
+
+ this.emf = emf;
this.entityManagerFig = entityManagerFig;
Preconditions.checkNotNull( cass, "cass must not be null" );
@@ -270,8 +259,6 @@ public class CpEntityManager implements EntityManager {
this.connectionService = connectionService;
this.collectionService = collectionService;
-
-
this.managerCache = managerCache;
this.applicationId = applicationId;
this.indexService = indexService;
@@ -755,6 +742,9 @@ public class CpEntityManager implements EntityManager {
@Override
public Application getApplication() throws Exception {
if ( application == null ) {
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( applicationId )) {
+ return emf.getManagementApplication();
+ }
application = get( applicationId, Application.class );
}
return application;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0c0c875/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 91a936d..b61be01 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -16,11 +16,7 @@
package org.apache.usergrid.corepersistence;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.UUID;
+import java.util.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -113,6 +109,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final ApplicationIdCache applicationIdCache;
//private final IndexSchemaCache indexSchemaCache;
+ Application managementApp = null;
+
private ManagerCache managerCache;
private CassandraService cassandraService;
@@ -126,8 +124,11 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
- public CpEntityManagerFactory( final CassandraService cassandraService, final CounterUtils counterUtils,
- final Injector injector ) {
+ public static final String MANAGEMENT_APP_MAX_RETRIES= "management.app.max.retries";
+
+
+ public CpEntityManagerFactory(
+ final CassandraService cassandraService, final CounterUtils counterUtils, final Injector injector ) {
this.cassandraService = cassandraService;
this.counterUtils = counterUtils;
@@ -142,12 +143,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
- //this line always needs to be last due to the temporary cicular dependency until spring is removed
+ // this line always needs to be last due to the temporary circular dependency until spring is removed
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
-
+ initMgmtAppInternal();
}
@@ -164,21 +165,61 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private void initMgmtAppInternal() {
+ Properties properties = cassandraService.getProperties();
+
+ Integer maxRetries;
+ try {
+ Object maxRetriesObject = properties.get( MANAGEMENT_APP_MAX_RETRIES ).toString();
+ maxRetries = Integer.parseInt( maxRetriesObject.toString() );
+ } catch ( NumberFormatException nfe ) {
+ maxRetries = 20;
+ }
+
EntityManager em = getEntityManager(getManagementAppId());
+
+ int retryCount = 0;
+
+ while ( managementApp != null && retryCount++ <= maxRetries ) {
+
+ try {
+ managementApp = em.getApplication();
+
+ if ( managementApp == null ) {
+
+ logger.warn( "Management application not found, attempting creation" );
+
+ Map mgmtAppProps = new HashMap<String, Object>();
+ mgmtAppProps.put( PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION );
+ em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps );
+ managementApp = em.getApplication();
+ }
+
+ } catch ( Throwable t ) {
+ logger.warn("Error getting or creating management application after " + retryCount + " retries", t);
+ }
+ }
+
indexService.queueInitializeApplicationIndex(CpNamingUtils.getApplicationScope(getManagementAppId()));
+ if ( managementApp == null ) {
+ throw new RuntimeException("FATAL ERROR: Failed to get or create management app");
+ }
+ }
+
+
+ public Application getManagementApplication() {
+
+ Application ret = null;
+ EntityManager em = getEntityManager(getManagementAppId());
try {
- if ( em.getApplication() == null ) {
- logger.info("Creating management application");
- Map mgmtAppProps = new HashMap<String, Object>();
- mgmtAppProps.put(PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION);
- em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps);
- em.getApplication();
- }
+ ret = em.getApplication();
+ managementApp = ret;
- } catch (Exception ex) {
- throw new RuntimeException("Fatal error creating management application", ex);
+ } catch (Exception e) {
+ logger.warn("Error getting management app, returning cached copy version");
}
+
+ return managementApp;
}
@@ -201,9 +242,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
- metricsFactory, entityManagerFig, graphManagerFactory, collectionService, connectionService,indexSchemaCacheFactory, applicationId );
-
+ EntityManager em = new CpEntityManager(
+ this,
+ cassandraService,
+ counterUtils,
+ indexService,
+ managerCache,
+ metricsFactory,
+ entityManagerFig,
+ graphManagerFactory,
+ collectionService,
+ connectionService,
+ indexSchemaCacheFactory,
+ applicationId );
return em;
}
@@ -215,7 +266,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Entity createApplicationV2(
- String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration) throws Exception {
+ String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration)
+ throws Exception {
String appName = buildAppName( orgName, name );
@@ -317,8 +369,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// find application_info for application to delete
- migrateAppInfo(applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS, CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking()
- .lastOrDefault( null );
+ migrateAppInfo(
+ applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS,
+ CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking().lastOrDefault( null );
}
//TODO: return status for restore
@@ -360,9 +413,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
* @return
* @throws Exception
*/
- private Observable migrateAppInfo(final UUID applicationUUID, final String deleteTypeName, final String createCollectionName, final String createTypeName ) throws Exception {
+ private Observable migrateAppInfo(
+ final UUID applicationUUID, final String deleteTypeName, final String createCollectionName,
+ final String createTypeName ) throws Exception {
+
+ final ApplicationScope managementAppScope =
+ CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
- final ApplicationScope managementAppScope = CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
final EntityManager managementEm = getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
//the application id we will be removing
@@ -417,7 +474,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final Id managementAppId = CpNamingUtils.getManagementApplicationId();
final EntityIndex aei = getManagementIndex();
final GraphManager managementGraphManager = managerCache.getGraphManager(managementAppScope);
- final Edge createEdge = CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
+ final Edge createEdge =
+ CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
final Observable createNodeGraph = managementGraphManager.writeEdge(createEdge);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e0c0c875/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index ef1baaf..993bf35 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -68,9 +68,6 @@ public class ServiceManager {
public static final String APPLICATION_REQUESTS_PER = APPLICATION_REQUESTS + ".";
public static final String IMPL = "Impl";
- public static final String SERVICE_MANAGER_RETRY_INTERVAL = "service.manager.retry.interval";
-
- public static final String SERVICE_MANAGER_MAX_RETRIES= "service.manager.max.retries";
private Application application;
@@ -99,69 +96,27 @@ public class ServiceManager {
this.qm = qm;
this.properties = properties;
- Integer retryInterval;
- try {
- Object retryIntervalObject = properties.get( SERVICE_MANAGER_RETRY_INTERVAL ).toString();
- retryInterval = Integer.parseInt( retryIntervalObject.toString() );
- } catch ( NumberFormatException nfe ) {
- retryInterval = 15000;
- }
-
- Integer maxRetries;
- try {
- Object maxRetriesObject = properties.get( SERVICE_MANAGER_MAX_RETRIES ).toString();
- maxRetries = Integer.parseInt( maxRetriesObject.toString() );
- } catch ( NumberFormatException nfe ) {
- maxRetries = 5;
- }
-
if ( em != null ) {
-
try {
- int retryCount = 0;
- boolean appNotFound = true;
-
- while ( appNotFound && retryCount <= maxRetries ) {
-
- application = em.getApplication();
-
- if ( application != null ) {
- appNotFound = false;
- applicationId = application.getUuid();
-
- } else {
- // Cassandra may be alive but responding very slowly, let's wait and retry
- logger.error("STARTUP PROBLEM: Cannot get application by UUID. Will retry in {} seconds #{}",
- retryInterval/1000, retryCount);
- Thread.sleep( retryInterval );
- retryCount++;
- }
- }
-
- if ( application == null ) {
- Exception e = new RuntimeException(
- "STARTUP FAILURE: application id {" + em.getApplicationId()
- + "} is returning null after " + retryCount + " retries" );
+ application = em.getApplication();
+ if(application == null){
+ Exception e = new RuntimeException("application id {"+em.getApplicationId()+"} is returning null");
+ logger.error("Failed to get application",e);
throw e;
}
-
- } catch ( RuntimeException re ) {
- logger.error( "ServiceManager init failure", re );
- throw re;
-
- } catch ( Exception e ) {
+ applicationId = application.getUuid();
+ }
+ catch ( Exception e ) {
logger.error( "ServiceManager init failure", e );
throw new RuntimeException( e );
}
}
-
if ( properties != null ) {
String packages = properties.getProperty( SERVICE_PACKAGE_PREFIXES );
if ( !StringUtils.isEmpty( packages ) ) {
setServicePackagePrefixes( packages );
}
}
-
return this;
}
[17/26] usergrid git commit: Make retries and interval configurable
Posted by mr...@apache.org.
Make retries and interval configurable
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/fcd00e83
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/fcd00e83
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/fcd00e83
Branch: refs/heads/release-2.1.1
Commit: fcd00e83b4febea1f395e7d8818bab8c38fa8bea
Parents: 910811d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 18:30:27 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 18:30:27 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 8 ++
.../corepersistence/CpEntityManagerFactory.java | 94 ++++++++++++--------
.../cassandra/AstyanaxLockManagerImpl.java | 5 +-
.../persistence/core/astyanax/CassandraFig.java | 18 +++-
4 files changed, 86 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 4f57cdd..1e22d6a 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -195,6 +195,12 @@ cassandra.lock.writecl=LOCAL_QUORUM
#
#cassandra.useSocketKeepalive=false
+# Number of times to retry creation of lock keyspace and column family
+cassandra.lock.init.retries = 100;
+
+# Interval between lock keyspace creation attempts (in milliseconds)
+cassandra.lock.init.interval = 1000;
+
##################### Async Threadpool Settings #####################
@@ -628,6 +634,8 @@ usergrid.auth.cache.inmemory.size=3000
# all (= in + out)'
usergrid.rest.default-connection-param=all
+# Number of times to retry attempt to access management app on startup
+management.app.max.retries=100
############################## Usergrid Testing #############################
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 84872aa..b6fbc2a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -23,6 +23,8 @@ import com.google.common.cache.LoadingCache;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
+import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
@@ -124,54 +126,79 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
+ Properties properties = cassandraService.getProperties();
+
+ entityManagers = createEntityManagerCache( properties );
+
+ checkManagementApp( properties );
+
+ // this line always needs to be last due to the temporary circular dependency until spring is removed
+ applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
+ .getInstance( getManagementEntityManager() );
+ }
+
+
+ private LoadingCache<UUID, EntityManager> createEntityManagerCache(Properties properties) {
+
int entityManagerCacheSize = 100;
try {
- entityManagerCacheSize = Integer.parseInt(
- cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ entityManagerCacheSize = Integer.parseInt( properties.getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
} catch ( Exception e ) {
- logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + ". Will use " + entityManagerCacheSize, e );
}
- entityManagers = CacheBuilder.newBuilder()
+ return CacheBuilder.newBuilder()
.maximumSize(entityManagerCacheSize)
.build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load( UUID appId ) { // no checked exception
+ public EntityManager load( UUID appId ) { // no checked exception
- // create new entity manager and pre-fetch its application
- EntityManager entityManager = _getEntityManager( appId );
- Application app = null;
- Exception exception = null;
- try {
- app = entityManager.getApplication();
- } catch (Exception e) {
- exception = e;
- }
+ // create new entity manager and pre-fetch its application
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Throwable throwable = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Throwable t) {
+ throwable = t;
+ }
- // the management app is a special case
+ // the management app is a special case
- if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
- if ( app != null ) {
- // we successfully fetched up the management app, cache it for a rainy day
- managementAppEntityManager = entityManager;
+ if ( app != null ) {
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
- } else if ( managementAppEntityManager != null ) {
- // failed to fetch management app, use cached one
- entityManager = managementAppEntityManager;
+ } else if ( managementAppEntityManager != null ) {
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+ }
+ }
+
+ if ( app == null && throwable != null && throwable.getCause() instanceof BadRequestException) {
+ // probably means schema has not been created yet
+ } else {
+ throw new RuntimeException( "Error getting application " + appId, throwable );
}
- }
- if (app == null) {
- throw new RuntimeException( "Error getting application " + appId, exception );
+ return entityManager;
}
+ });
+ }
- return entityManager;
- }
- });
+
+ private void checkManagementApp(Properties properties) {
+ int maxRetries = 100;
+ try {
+ maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES, "100" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " + maxRetries, e );
+ }
// hold up construction until we can access the management app
- int maxRetries = 1000;
int retries = 0;
boolean managementAppFound = false;
while ( !managementAppFound && retries++ < maxRetries ) {
@@ -186,6 +213,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
} else {
logger.error(msg);
}
+ try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {}
}
}
@@ -193,10 +221,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// exception here will prevent WAR from being deployed
throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
}
-
- // this line always needs to be last due to the temporary circular dependency until spring is removed
- applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
- .getInstance( getManagementEntityManager() );
}
@@ -240,8 +264,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
try {
return entityManagers.get( applicationId );
}
- catch ( Exception ex ) {
- logger.error("Error getting oldAppInfo manager", ex);
+ catch ( Throwable t ) {
+ logger.error("Error getting entity manager", t);
}
return _getEntityManager(applicationId);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 49ff52e..9bf6694 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -52,6 +52,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
private ColumnFamily columnFamily;
private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute
+
@Inject
public AstyanaxLockManagerImpl(CassandraFig cassandraFig,
CassandraCluster cassandraCluster ) throws ConnectionException {
@@ -59,7 +60,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
this.cassandraFig = cassandraFig;
// hold up construction until we can create the column family
- int maxRetries = 1000;
+ int maxRetries = cassandraFig.getLockManagerInitRetries();
int retries = 0;
boolean famReady = false;
while ( !famReady && retries++ < maxRetries ) {
@@ -76,7 +77,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
} else {
logger.error( msg );
}
- try { Thread.sleep(1000); } catch (InterruptedException ignored) {}
+ try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch (InterruptedException ignored) {}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/fcd00e83/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
index b00eca8..d315561 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraFig.java
@@ -50,8 +50,8 @@ public interface CassandraFig extends GuicyFig {
String LOCKS_CONNECTIONS = "cassandra.lock.connections";
String LOCKS_EXPIRATION = "cassandra.lock.expiration.milliseconds";
-
-
+ String LOCK_MANAGER_INIT_RETRIES = "cassandra.lock.init.retries";
+ String LOCK_MANAGER_INIT_INTERVAL = "cassandra.lock.init.interval";
// re-usable default values
String DEFAULT_CONNECTION_POOLSIZE = "15";
@@ -180,4 +180,18 @@ public interface CassandraFig extends GuicyFig {
@Default(DEFAULT_LOCKS_EXPIRATION)
int getLocksExpiration();
+ /**
+ * How many times to attempt lock keyspace and column family creation
+ */
+ @Key( LOCK_MANAGER_INIT_RETRIES )
+ @Default( "100" )
+ int getLockManagerInitRetries();
+
+ /**
+ * Return the expiration that should be used for expiring a lock if it's not released
+ */
+ @Key( LOCK_MANAGER_INIT_INTERVAL )
+ @Default( "1000" )
+ int getLockManagerInitInterval();
+
}
[09/26] usergrid git commit: Improvements to error handling in cache
key generation
Posted by mr...@apache.org.
Improvements to error handling in cache key generation
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ed4e67c8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ed4e67c8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ed4e67c8
Branch: refs/heads/release-2.1.1
Commit: ed4e67c855a1d1062d0cfa5976d05db6b044f11a
Parents: aae8fdf
Author: Dave Johnson <sn...@apache.org>
Authored: Mon May 23 11:41:44 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Mon May 23 11:41:44 2016 -0400
----------------------------------------------------------------------
.../usergrid/security/shiro/ShiroCache.java | 80 ++++++++++++--------
1 file changed, 49 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ed4e67c8/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
index b4803b1..6c5ac6c 100644
--- a/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
+++ b/stack/services/src/main/java/org/apache/usergrid/security/shiro/ShiroCache.java
@@ -184,51 +184,69 @@ public class ShiroCache<K, V> implements Cache<K,V> {
String ret = null;
- final String typeName = typeRef.getType().getTypeName();
+ Throwable throwable = null;
- if ( key instanceof SimplePrincipalCollection) {
+ String errorMessage = null;
- SimplePrincipalCollection spc = (SimplePrincipalCollection)key;
+ try {
- if ( spc.getPrimaryPrincipal() instanceof UserPrincipal) {
+ final String typeName = typeRef.getType().getTypeName();
- // principal is a user, use UUID as cache key
- UserPrincipal p = (UserPrincipal) spc.getPrimaryPrincipal();
- ret = p.getUser().getUuid().toString() + "_" + typeName;
- }
+ if (key instanceof SimplePrincipalCollection) {
+
+ SimplePrincipalCollection spc = (SimplePrincipalCollection) key;
+
+ if (spc.getPrimaryPrincipal() instanceof UserPrincipal) {
+
+ // principal is a user, use UUID as cache key
+ UserPrincipal p = (UserPrincipal) spc.getPrimaryPrincipal();
+ ret = p.getUser().getUuid().toString() + "_" + typeName;
+
+ } else if (spc.getPrimaryPrincipal() instanceof PrincipalIdentifier) {
- else if ( spc.getPrimaryPrincipal() instanceof PrincipalIdentifier ) {
+ // principal is not user, try to get something unique as cache key
+ PrincipalIdentifier p = (PrincipalIdentifier) spc.getPrimaryPrincipal();
+ if (p.getAccessTokenCredentials() != null) {
+ ret = p.getAccessTokenCredentials().getToken() + "_" + typeName;
+ } else {
+ ret = p.getApplicationId() + "_" + typeName;
+ }
- // principal is not user, try to get something unique as cache key
- PrincipalIdentifier p = (PrincipalIdentifier) spc.getPrimaryPrincipal();
- if (p.getAccessTokenCredentials() != null) {
- ret = p.getAccessTokenCredentials().getToken() + "_" + typeName;
} else {
- ret = p.getApplicationId() + "_" + typeName;
+ errorMessage = "Unknown principal type: " + key.getClass().getSimpleName();
}
- }
- } else if ( key instanceof ApplicationGuestPrincipal ) {
- ApplicationGuestPrincipal agp = (ApplicationGuestPrincipal) key;
- ret = agp.getApplicationId() + "_" + typeName;
+ } else if (key instanceof ApplicationGuestPrincipal) {
+ ApplicationGuestPrincipal agp = (ApplicationGuestPrincipal) key;
+ ret = agp.getApplicationId() + "_" + typeName;
+
+ } else if (key instanceof ApplicationPrincipal) {
+ ApplicationPrincipal ap = (ApplicationPrincipal) key;
+ ret = ap.getApplicationId() + "_" + typeName;
+
+ } else if (key instanceof OrganizationPrincipal) {
+ OrganizationPrincipal op = (OrganizationPrincipal) key;
+ ret = op.getOrganizationId() + "_" + typeName;
- } else if ( key instanceof ApplicationPrincipal ) {
- ApplicationPrincipal ap = (ApplicationPrincipal) key;
- ret = ap.getApplicationId() + "_" + typeName;
+ } else if (key instanceof UserPrincipal) {
+ UserPrincipal up = (UserPrincipal) key;
+ ret = up.getUser().getUuid() + "_" + typeName;
- } else if ( key instanceof OrganizationPrincipal ) {
- OrganizationPrincipal op = (OrganizationPrincipal) key;
- ret = op.getOrganizationId() + "_" + typeName;
+ } else {
+ errorMessage = "Unknown key type: " + key.getClass().getSimpleName();
+ }
+
+ } catch ( Throwable t ) {
+ throwable = t;
+ }
- } else if ( key instanceof UserPrincipal ) {
- UserPrincipal up = (UserPrincipal)key;
- ret = up.getUser().getUuid() + "_" + typeName;
+ if ( throwable != null ) {
+ errorMessage = "Error generating cache key for key type " + key.getClass().getSimpleName();
+ throw new CacheException( errorMessage, throwable );
}
- if ( ret == null) {
- String msg = "Unknown key type: " + key.getClass().getSimpleName();
- logger.error(msg);
- throw new RuntimeException(msg);
+ if ( ret == null ) {
+ throw new CacheException( errorMessage );
}
return ret;
[08/26] usergrid git commit: Minor refactoring and formatting changes.
Posted by mr...@apache.org.
Minor refactoring and formatting changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e9e7c39c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e9e7c39c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e9e7c39c
Branch: refs/heads/release-2.1.1
Commit: e9e7c39c46ca0b49b5891ee04bdac648e9777cee
Parents: 0b41620
Author: Dave Johnson <sn...@apache.org>
Authored: Fri May 20 16:00:21 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri May 20 16:00:21 2016 -0400
----------------------------------------------------------------------
.../pipeline/read/search/CandidateEntityFilter.java | 12 +++++++++---
.../java/org/apache/usergrid/persistence/IndexIT.java | 2 +-
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e9e7c39c/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index e02e6c7..4aa6c8d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -163,17 +163,19 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
+
if ( fieldMap.containsKey( parts[0] )) {
Field field = fieldMap.get( parts[0] );
if ( field instanceof EntityObjectField ) {
EntityObjectField eof = (EntityObjectField)field;
- if ( result.get( parts[0] ) == null ) {
- result.put( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
- }
+ result.putIfAbsent( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
+
+ // recursion
nestedFieldSet(
((EntityObjectField)result.get( parts[0] )).getValue().getFieldMap(),
Arrays.copyOfRange(parts, 1, parts.length),
eof.getValue().getFieldMap());
+
} else {
result.put( parts[0], field );
}
@@ -184,11 +186,15 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
+
if ( fieldMap.containsKey( parts[0] )) {
Field field = fieldMap.get( parts[0] );
if ( field instanceof EntityObjectField ) {
EntityObjectField eof = (EntityObjectField)field;
+
+ // recursion
return nestedFieldCheck( Arrays.copyOfRange(parts, 1, parts.length), eof.getValue().getFieldMap());
+
} else {
return true;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e9e7c39c/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index aaf4c33..d62f88e 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -528,7 +528,7 @@ public class IndexIT extends AbstractCoreIT {
assertTrue( first.getDynamicProperties().size() == 3 );
}
- // multiple nested fields with one doubly-nesed field
+ // multiple nested fields with one doubly-nested field
{
Query query = Query.fromQL( "select data.rando, data.mondo, data.misc.range where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
[04/26] usergrid git commit: Initial support for select mappings with
entity object fields (i.e. nested fields)
Posted by mr...@apache.org.
Initial support for select mappings with entity object fields (i.e. nested fields)
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/596e0fc5
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/596e0fc5
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/596e0fc5
Branch: refs/heads/release-2.1.1
Commit: 596e0fc5cef2eb1ae4aba40343bf27e73aee86d6
Parents: 7fdca3d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 17:24:25 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 17:24:25 2016 -0400
----------------------------------------------------------------------
.../read/search/CandidateEntityFilter.java | 59 ++++++++++++++-
.../apache/usergrid/persistence/IndexIT.java | 80 ++++++++++++++++++++
2 files changed, 135 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/596e0fc5/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index d47e96c..261259b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -27,7 +27,9 @@ import org.apache.usergrid.persistence.index.*;
import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.persistence.model.field.DistanceField;
import org.apache.usergrid.persistence.model.field.DoubleField;
+import org.apache.usergrid.persistence.model.field.EntityObjectField;
import org.apache.usergrid.persistence.model.field.Field;
+import org.apache.usergrid.persistence.model.field.value.EntityObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,12 +125,26 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
if (mappings.size() > 0) {
Map<String,Field> fieldMap = new HashMap<String, Field>(mappings.size());
rx.Observable.from(mappings)
- .filter(mapping -> entity.getFieldMap().containsKey(mapping.getSourceFieldName()))
+
+ .filter(mapping -> {
+ if ( entity.getFieldMap().containsKey(mapping.getSourceFieldName())) {
+ return true;
+ }
+ String[] parts = mapping.getSourceFieldName().split("\\.");
+ return nestedFieldCheck( parts, entity.getFieldMap() );
+ })
+
.doOnNext(mapping -> {
Field field = entity.getField(mapping.getSourceFieldName());
- field.setName(mapping.getTargetFieldName());
- fieldMap.put(mapping.getTargetFieldName(),field);
- }).toBlocking().last();
+ if ( field != null ) {
+ field.setName( mapping.getTargetFieldName() );
+ fieldMap.put( mapping.getTargetFieldName(), field );
+ } else {
+ String[] parts = mapping.getSourceFieldName().split("\\.");
+ nestedFieldSet( fieldMap, parts, entity.getFieldMap() );
+ }
+ }).toBlocking().lastOrDefault(null);
+
entity.setFieldMap(fieldMap);
}
return entityFilterResult;
@@ -144,6 +160,41 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
}
+ private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
+ if ( parts.length > 0 ) {
+ if ( fieldMap.containsKey( parts[0] )) {
+ Field field = fieldMap.get( parts[0] );
+ if ( field instanceof EntityObjectField ) {
+ EntityObjectField eof = (EntityObjectField)field;
+ if ( result.get( parts[0] ) == null ) {
+ result.put( parts[0], new EntityObjectField( parts[0], new EntityObject() ) );
+ }
+ nestedFieldSet(
+ ((EntityObjectField)result.get( parts[0] )).getValue().getFieldMap(),
+ Arrays.copyOfRange(parts, 1, parts.length),
+ eof.getValue().getFieldMap());
+ } else {
+ result.put( parts[0], field );
+ }
+ }
+ }
+ }
+
+
+ private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
+ if ( parts.length > 0 ) {
+ if ( fieldMap.containsKey( parts[0] )) {
+ Field field = fieldMap.get( parts[0] );
+ if ( field instanceof EntityObjectField ) {
+ EntityObjectField eof = (EntityObjectField)field;
+ return nestedFieldCheck( Arrays.copyOfRange(parts, 1, parts.length), eof.getValue().getFieldMap());
+ } else {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
/**
http://git-wip-us.apache.org/repos/asf/usergrid/blob/596e0fc5/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index f4aa204..7e38f17 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -17,6 +17,7 @@
package org.apache.usergrid.persistence;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.UUID;
@@ -461,4 +462,83 @@ public class IndexIT extends AbstractCoreIT {
}
+
+ @Test
+ public void testSelectMappings() throws Exception {
+
+ UUID applicationId = app.getId();
+
+ EntityManager em = setup.getEmf().getEntityManager(applicationId);
+
+ Map<String, Object> entity1 = new HashMap<String, Object>() {{
+ put("name","name_1");
+ put("status", "pickled");
+ put("data", new HashMap() {{
+ put("xfactor", 5.1);
+ put("rando", "bar");
+ }});
+ }};
+ em.create("names", entity1);
+
+ Map<String, Object> entity2 = new HashMap<String, Object>() {{
+ put("name","name_2");
+ put("status", "pickled");
+ put("data", new HashMap() {{
+ put("xfactor", 5.1);
+ put("rando", "bar");
+ }});
+ }};
+ em.create("names", entity2);
+
+ app.refreshIndex();
+
+ {
+ Query query = Query.fromQL("select status where status = 'pickled'");
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue(r.getEntities() != null && r.getEntities().size() == 2);
+ Entity first = r.getEntities().get(0);
+ assertTrue(first.getDynamicProperties().size() == 2);
+ }
+
+ {
+ Query query = Query.fromQL( "select status, data.rando where data.rando = 'bar'" );
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+
+ Entity first = r.getEntities().get( 0 );
+
+ assertNotNull( first.getProperty("status") );
+ assertEquals( first.getProperty("status"), "pickled" );
+
+ assertNotNull( first.getProperty("data") );
+ assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+
+ assertTrue( first.getDynamicProperties().size() == 3 );
+ }
+
+ {
+ // query for only one bogus field name should return empty entities
+ Query query = Query.fromQL( "select data.rando where status = 'pickled'" );
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+
+ Entity first = r.getEntities().get( 0 );
+
+ assertNotNull( first.getProperty("data") );
+ assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+
+ assertTrue( first.getDynamicProperties().size() == 2 );
+ }
+
+ {
+ // query for only one bogus field name should return empty entities
+ Query query = Query.fromQL( "select data.bogusfieldname where status = 'pickled'" );
+ Results r = em.searchCollection( em.getApplicationRef(), "names", query );
+ assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
+ Entity first = r.getEntities().get( 0 );
+ assertTrue( first.getDynamicProperties().size() == 1 );
+ }
+
+ }
+
}
[06/26] usergrid git commit: Test for doubly nested field,
plus some formatting changes.
Posted by mr...@apache.org.
Test for doubly nested field, plus some formatting changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/0b416203
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/0b416203
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/0b416203
Branch: refs/heads/release-2.1.1
Commit: 0b4162035ee7d54199bd2250c2b3ac31354516b1
Parents: 3eec8e5
Author: Dave Johnson <sn...@apache.org>
Authored: Thu May 19 08:57:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu May 19 08:57:42 2016 -0400
----------------------------------------------------------------------
.../read/search/CandidateEntityFilter.java | 16 +++++++-----
.../apache/usergrid/persistence/IndexIT.java | 27 +++++++++++++++-----
2 files changed, 30 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0b416203/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 261259b..e02e6c7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -52,9 +52,9 @@ import rx.Observable;
/**
- * Loads entities from an incoming CandidateResult emissions into entities, then streams them on
- * performs internal buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries. It is up to the
- * collector to determine when to stop streaming entities.
+ * Loads entities from an incoming CandidateResult emissions into entities, then streams them on performs internal
+ * buffering for efficiency. Note that all entities may not be emitted if our load crosses page boundaries.
+ * It is up to the collector to determine when to stop streaming entities.
*/
public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate>, FilterResult<Entity>> {
@@ -93,11 +93,12 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
entityCollectionManagerFactory.createCollectionManager( applicationScope );
- final EntityIndex applicationIndex =
- entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
+ final EntityIndex applicationIndex = entityIndexFactory
+ .createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
//buffer them to get a page size we can make 1 network hop
- final Observable<FilterResult<Entity>> searchIdSetObservable = candidateResultsObservable.buffer( pipelineContext.getLimit() )
+ final Observable<FilterResult<Entity>> searchIdSetObservable =
+ candidateResultsObservable.buffer( pipelineContext.getLimit() )
//load them
.flatMap( candidateResults -> {
@@ -198,7 +199,8 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
/**
- * Our collector to collect entities. Not quite a true collector, but works within our operational flow as this state is mutable and difficult to represent functionally
+ * Our collector to collect entities. Not quite a true collector, but works within our operational
+ * flow as this state is mutable and difficult to represent functionally
*/
private static final class EntityVerifier {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/0b416203/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index 3b9f95a..aaf4c33 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -477,6 +477,10 @@ public class IndexIT extends AbstractCoreIT {
put("xfactor", 5.1);
put("rando", "bar");
put("mondo", "2000");
+ put("frosting", "chocolate");
+ put("misc", new HashMap() {{
+ put("range", "open");
+ }});
}});
}};
em.create("names", entity1);
@@ -488,12 +492,17 @@ public class IndexIT extends AbstractCoreIT {
put("xfactor", 5.1);
put("rando", "bar");
put("mondo", "2001");
+ put("frosting", "orange");
+ put("misc", new HashMap() {{
+ put("range", "open");
+ }});
}});
}};
em.create("names", entity2);
app.refreshIndex();
+ // simple single-field select mapping
{
Query query = Query.fromQL("select status where status = 'pickled'");
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
@@ -502,6 +511,7 @@ public class IndexIT extends AbstractCoreIT {
assertTrue(first.getDynamicProperties().size() == 2);
}
+ // simple single-field plus nested field select mapping
{
Query query = Query.fromQL( "select status, data.rando where data.rando = 'bar'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
@@ -518,23 +528,28 @@ public class IndexIT extends AbstractCoreIT {
assertTrue( first.getDynamicProperties().size() == 3 );
}
+ // multiple nested fields with one doubly-nesed field
{
- // query for only one bogus field name should return empty entities
- Query query = Query.fromQL( "select data.rando,data.mondo where status = 'pickled'" );
+ Query query = Query.fromQL( "select data.rando, data.mondo, data.misc.range where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
Entity first = r.getEntities().get( 0 );
- assertNotNull( first.getProperty("data") );
- assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
- assertEquals( ((Map<String, Object>)first.getProperty("data")).get("mondo"), "2001" );
+ Map<String, Object> data = ((Map<String, Object>)first.getProperty("data"));
+ assertNotNull( data );
+ assertEquals( data.get("rando"), "bar" );
+ assertEquals( data.get("mondo"), "2001" );
+ assertNull( data.get("frosting") );
+
+ Map<String, Object> misc = (Map<String, Object>)data.get("misc");
+ assertEquals( misc.get("range"), "open" );
assertTrue( first.getDynamicProperties().size() == 2 );
}
+ // query for one bogus field name should return empty entities
{
- // query for only one bogus field name should return empty entities
Query query = Query.fromQL( "select data.bogusfieldname where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
[20/26] usergrid git commit: Merge branch 'release-2.1.1' into
select-mappings-nested
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' into select-mappings-nested
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/151abf78
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/151abf78
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/151abf78
Branch: refs/heads/release-2.1.1
Commit: 151abf78f92cebd2ac8701d18b9754f65b88cfdc
Parents: e9e7c39 0970e1d
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 31 13:32:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 31 13:32:42 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 --
.../applications/ApplicationResourceIT.java | 5 +-
.../usergrid/security/shiro/ShiroCache.java | 44 +++++++++++++-
.../usergrid/services/ServiceManager.java | 61 +++-----------------
4 files changed, 52 insertions(+), 63 deletions(-)
----------------------------------------------------------------------
[07/26] usergrid git commit: Revert changes to initMgmtApp() and add
null-pointer check.
Posted by mr...@apache.org.
Revert changes to initMgmtApp() and add null-pointer check.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/12743f3e
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/12743f3e
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/12743f3e
Branch: refs/heads/release-2.1.1
Commit: 12743f3e221f80df7eab89fda08325e9775f7ae9
Parents: de6de66
Author: Dave Johnson <sn...@apache.org>
Authored: Fri May 20 14:03:53 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri May 20 14:03:53 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 49 +++++---------------
1 file changed, 12 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/12743f3e/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e057210..924dd10 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -175,12 +175,12 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
- if ( app != null ) {
+ if ( app != null && entityManager != null ) {
// we successfully fetched up the management app, cache it for a rainy day
managementAppEntityManager = entityManager;
- } else if ( managementAppEntityManager != null ) {
+ } else if ( entityManager == null && managementAppEntityManager != null ) {
// failed to fetch management app, use cached one
entityManager = managementAppEntityManager;
@@ -214,47 +214,22 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
-
private void initMgmtAppInternal() {
- Properties properties = cassandraService.getProperties();
-
- Integer maxRetries;
- try {
- Object maxRetriesObject = properties.get( MANAGEMENT_APP_MAX_RETRIES ).toString();
- maxRetries = Integer.parseInt( maxRetriesObject.toString() );
- } catch ( NumberFormatException nfe ) {
- maxRetries = 20;
- }
-
EntityManager em = getEntityManager(getManagementAppId());
+ indexService.queueInitializeApplicationIndex(CpNamingUtils.getApplicationScope(getManagementAppId()));
- int retryCount = 0;
-
- while ( managementApp != null && retryCount++ <= maxRetries ) {
-
- try {
- managementApp = em.getApplication();
-
- if ( managementApp == null ) {
-
- logger.warn( "Management application not found, attempting creation" );
-
- Map mgmtAppProps = new HashMap<String, Object>();
- mgmtAppProps.put( PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION );
- em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps );
- managementApp = em.getApplication();
- }
-
- } catch ( Throwable t ) {
- logger.warn("Error getting or creating management application after " + retryCount + " retries", t);
+ try {
+ if ( em.getApplication() == null ) {
+ logger.info("Creating management application");
+ Map mgmtAppProps = new HashMap<String, Object>();
+ mgmtAppProps.put(PROPERTY_NAME, CassandraService.MANAGEMENT_APPLICATION);
+ em.create( getManagementAppId(), TYPE_APPLICATION, mgmtAppProps);
+ em.getApplication();
}
- }
-
- indexService.queueInitializeApplicationIndex(CpNamingUtils.getApplicationScope(getManagementAppId()));
- if ( managementApp == null ) {
- throw new RuntimeException("FATAL ERROR: Failed to get or create management app");
+ } catch (Exception ex) {
+ throw new RuntimeException("Fatal error creating management application", ex);
}
}
[15/26] usergrid git commit: Revert all changes to
CpEntityManagerFactory
Posted by mr...@apache.org.
Revert all changes to CpEntityManagerFactory
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/ba10e7f9
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/ba10e7f9
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/ba10e7f9
Branch: refs/heads/release-2.1.1
Commit: ba10e7f94a4b55bb0c89fc209d14a7a5687760fe
Parents: b0fba68
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:29:57 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:29:57 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 84 ++++++--------------
1 file changed, 25 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/ba10e7f9/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index f92fc9e..d2417be 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -16,16 +16,14 @@
package org.apache.usergrid.corepersistence;
-import java.util.*;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.BeansException;
-import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationContextAware;
-
+import com.google.common.base.Optional;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Injector;
+import com.google.inject.Key;
+import com.google.inject.TypeLiteral;
import org.apache.commons.lang.StringUtils;
-
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
import org.apache.usergrid.corepersistence.index.ReIndexRequestBuilder;
@@ -34,15 +32,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
-import org.apache.usergrid.persistence.AbstractEntity;
-import org.apache.usergrid.persistence.Entity;
-import org.apache.usergrid.persistence.EntityFactory;
-import org.apache.usergrid.persistence.EntityManager;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.EntityRef;
-import org.apache.usergrid.persistence.Query;
-import org.apache.usergrid.persistence.Results;
-import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
@@ -57,30 +47,23 @@ import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
import org.apache.usergrid.persistence.exceptions.EntityNotFoundException;
-import org.apache.usergrid.persistence.graph.Edge;
-import org.apache.usergrid.persistence.graph.GraphManager;
-import org.apache.usergrid.persistence.graph.GraphManagerFactory;
-import org.apache.usergrid.persistence.graph.MarkedEdge;
-import org.apache.usergrid.persistence.graph.SearchByEdgeType;
+import org.apache.usergrid.persistence.graph.*;
import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.apache.usergrid.utils.UUIDUtils;
-
-import com.google.common.base.Optional;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Injector;
-import com.google.inject.Key;
-import com.google.inject.TypeLiteral;
-
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.BeansException;
+import org.springframework.context.ApplicationContext;
+import org.springframework.context.ApplicationContextAware;
import rx.Observable;
-import static java.lang.String.CASE_INSENSITIVE_ORDER;
+import java.util.*;
+import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -265,19 +248,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
- EntityManager em = new CpEntityManager(
- this,
- cassandraService,
- counterUtils,
- indexService,
- managerCache,
- metricsFactory,
- entityManagerFig,
- graphManagerFactory,
- collectionService,
- connectionService,
- indexSchemaCacheFactory,
- applicationId );
+ EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
+ metricsFactory, entityManagerFig, graphManagerFactory, collectionService, connectionService,indexSchemaCacheFactory, applicationId );
+
return em;
}
@@ -289,8 +262,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public Entity createApplicationV2(
- String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration)
- throws Exception {
+ String orgName, String name, UUID applicationId, Map<String, Object> properties, boolean forMigration) throws Exception {
String appName = buildAppName( orgName, name );
@@ -392,9 +364,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// find application_info for application to delete
- migrateAppInfo(
- applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS,
- CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking().lastOrDefault( null );
+ migrateAppInfo(applicationId, CpNamingUtils.APPLICATION_INFO, CpNamingUtils.DELETED_APPLICATION_INFOS, CpNamingUtils.DELETED_APPLICATION_INFO).toBlocking()
+ .lastOrDefault( null );
}
//TODO: return status for restore
@@ -436,13 +407,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
* @return
* @throws Exception
*/
- private Observable migrateAppInfo(
- final UUID applicationUUID, final String deleteTypeName, final String createCollectionName,
- final String createTypeName ) throws Exception {
-
- final ApplicationScope managementAppScope =
- CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
+ private Observable migrateAppInfo(final UUID applicationUUID, final String deleteTypeName, final String createCollectionName, final String createTypeName ) throws Exception {
+ final ApplicationScope managementAppScope = CpNamingUtils.getApplicationScope(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
final EntityManager managementEm = getEntityManager(CpNamingUtils.MANAGEMENT_APPLICATION_ID);
//the application id we will be removing
@@ -497,8 +464,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
final Id managementAppId = CpNamingUtils.getManagementApplicationId();
final EntityIndex aei = getManagementIndex();
final GraphManager managementGraphManager = managerCache.getGraphManager(managementAppScope);
- final Edge createEdge =
- CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
+ final Edge createEdge = CpNamingUtils.createCollectionEdge(managementAppId, createCollectionName, createApplicationId);
final Observable createNodeGraph = managementGraphManager.writeEdge(createEdge);
[03/26] usergrid git commit: Merge branch
'usergrid-1283-service-mgr-init' into usegrid-1283-mgmt-app-init
Posted by mr...@apache.org.
Merge branch 'usergrid-1283-service-mgr-init' into usegrid-1283-mgmt-app-init
Conflicts:
stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/de6de662
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/de6de662
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/de6de662
Branch: refs/heads/release-2.1.1
Commit: de6de6629903f57df4faf5c3f20f5cea63f2bcec
Parents: e0c0c87 5f46341
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 09:23:54 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 09:23:54 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 66 +++++++++++++++++---
1 file changed, 59 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/de6de662/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b61be01,4028875..e057210
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -98,14 -102,15 +98,12 @@@ public class CpEntityManagerFactory imp
private Setup setup = null;
+ EntityManager managementAppEntityManager = null;
+
// cache of already instantiated entity managers
- private LoadingCache<UUID, EntityManager> entityManagers
- = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load(UUID appId) { // no checked exception
- return _getEntityManager(appId);
- }
- });
+ private final String ENTITY_MANAGER_CACHE_SIZE = "entity.manager.cache.size";
+ private final LoadingCache<UUID, EntityManager> entityManagers;
-
-
-
private final ApplicationIdCache applicationIdCache;
//private final IndexSchemaCache indexSchemaCache;
@@@ -148,7 -148,62 +146,61 @@@
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
- initMgmtAppInternal();
-
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt(
+ cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ }
+
+ entityManagers = CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+
+ public EntityManager load( UUID appId ) { // no checked exception
+
+ // get entity manager and ensure it can get its own application
+
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Exception exception = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Exception e) {
+ exception = e;
+ }
+
+ // the management app is a special case
+
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
+ if ( app != null ) {
+
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
+ } else if ( managementAppEntityManager != null ) {
+
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+
+ } else {
+
+ // fetch failed and we have nothing cached, we must be bootstrapping
+ logger.info("managementAppEntityManager is null, bootstrapping in progress");
+ }
+
+ } else { // not the management app, so blow up if app cannot be fetched
+
+ if (app == null) {
+ throw new RuntimeException( "Error getting application " + appId, exception );
+ }
+ }
+
+ return entityManager;
+ }
+ });
}
[18/26] usergrid git commit: Made max-retries and intervals
configurable for both lock manager and entity manager factory,
also better error logging.
Posted by mr...@apache.org.
Made max-retries and intervals configurable for both lock manager and entity manager factory, also better error logging.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61a35a04
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61a35a04
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61a35a04
Branch: refs/heads/release-2.1.1
Commit: 61a35a041598af0e336199343ed6a489454980f6
Parents: fcd00e8
Author: Dave Johnson <sn...@apache.org>
Authored: Thu May 26 11:10:33 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu May 26 11:10:33 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 83 ++++++++++++++++----
.../cassandra/AstyanaxLockManagerImpl.java | 17 +++-
.../exception/CollectionRuntimeException.java | 11 +++
3 files changed, 93 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b6fbc2a..ee28765 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -20,11 +20,10 @@ import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.UncheckedExecutionException;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
-import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -68,6 +68,7 @@ import java.util.*;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.initLock;
/**
@@ -107,7 +108,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
- public static final String MANAGEMENT_APP_MAX_RETRIES= "management.app.max.retries";
+ public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
+ public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
public CpEntityManagerFactory(
@@ -177,12 +179,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
- if ( app == null && throwable != null && throwable.getCause() instanceof BadRequestException) {
- // probably means schema has not been created yet
+ final boolean missingKeyspace;
+ if ( throwable instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
+ missingKeyspace = cre.isMissingKeyspace();
} else {
- throw new RuntimeException( "Error getting application " + appId, throwable );
+ missingKeyspace = false;
}
+ if ( app == null && !missingKeyspace ) {
+ throw new RuntimeException( "Error getting application " + appId, throwable );
+
+ } // else keyspace is missing because setup/bootstrap not done yet
+
return entityManager;
}
});
@@ -190,34 +199,77 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private void checkManagementApp(Properties properties) {
+
int maxRetries = 100;
try {
- maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES, "100" ));
+ maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_MAXRETRIES, "100" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_MAXRETRIES + ". Will use " + maxRetries, e );
+ }
+
+ int interval = 1000;
+ try {
+ interval = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_INTERVAL, "1000" ));
} catch ( Exception e ) {
- logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " + maxRetries, e );
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_INTERVAL + ". Will use " + maxRetries, e );
}
// hold up construction until we can access the management app
int retries = 0;
boolean managementAppFound = false;
+ boolean bootstrapping = false;
+ Set<Class> seenBefore = new HashSet<>(10);
while ( !managementAppFound && retries++ < maxRetries ) {
try {
- getEntityManager( getManagementAppId() ).getApplication();
+ // bypass entity manager cache and get managementApp
+ managementApp = _getEntityManager( getManagementAppId() ).getApplication();
managementAppFound = true;
} catch ( Throwable t ) {
- String msg = "Error " + t.getClass() + " getting management app on try " + retries;
- if ( logger.isDebugEnabled() ) {
+
+ if ( t instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException)t;
+ if ( cre.isMissingKeyspace() ) {
+ // we're bootstrapping, ignore this and continue
+ bootstrapping = true;
+ break;
+ }
+ }
+ Throwable cause = t;
+
+ // there was an error, be as informative as possible
+ StringBuilder sb = new StringBuilder();
+ sb.append(retries).append(": Error (");
+
+ if ( t instanceof UncheckedExecutionException ) {
+ UncheckedExecutionException uee = (UncheckedExecutionException)t;
+ if ( uee.getCause() instanceof RuntimeException ) {
+ cause = uee.getCause().getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ")
+ .append(uee.getCause().getMessage());
+ } else {
+ cause = uee.getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+ } else {
+ sb.append(t.getCause().getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+
+ String msg = sb.toString();
+ if ( !seenBefore.contains( cause.getClass() ) ) {
logger.error( msg, t);
} else {
logger.error(msg);
}
- try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {}
+ seenBefore.add( cause.getClass() );
+
+ try { Thread.sleep( interval ); } catch (InterruptedException ignored) {}
}
}
- if ( !managementAppFound ) {
+ if ( !managementAppFound && !bootstrapping ) {
// exception here will prevent WAR from being deployed
throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
}
@@ -323,8 +375,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// Ensure the management application is initialized
initMgmtAppInternal();
- EntityManager managementEm = getEntityManager( getManagementAppId() );
- EntityManager appEm = getEntityManager(applicationId);
+ // Get entity managers by bypassing the entity manager cache because it expects apps to already exist
+ final EntityManager managementEm = _getEntityManager( getManagementAppId() );
+ EntityManager appEm = _getEntityManager(applicationId);
final String appName = buildAppName(organizationName, name);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 9bf6694..6acce47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -22,6 +22,8 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.connectionpool.exceptions.NoAvailableHostsException;
+import com.netflix.astyanax.connectionpool.exceptions.PoolTimeoutException;
import com.netflix.astyanax.ddl.ColumnFamilyDefinition;
import com.netflix.astyanax.ddl.KeyspaceDefinition;
import com.netflix.astyanax.model.ColumnFamily;
@@ -63,6 +65,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
int maxRetries = cassandraFig.getLockManagerInitRetries();
int retries = 0;
boolean famReady = false;
+ Set<Class> seenBefore = new HashSet<>(10);
while ( !famReady && retries++ < maxRetries ) {
try {
keyspace = cassandraCluster.getLocksKeyspace();
@@ -71,13 +74,21 @@ public class AstyanaxLockManagerImpl implements LockManager {
famReady = true;
} catch ( Throwable t ) {
- String msg = "Error " + t.getClass().getSimpleName() + " creating locks keyspace try " + retries;
- if ( logger.isDebugEnabled() ) {
+ final String msg;
+ if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException) {
+ msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName() + ")";
+ } else {
+ msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries=" + retries;
+ }
+ if ( !seenBefore.contains( t.getClass() ) ) {
logger.error( msg, t );
} else {
logger.error( msg );
}
- try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch (InterruptedException ignored) {}
+ seenBefore.add( t.getClass() );
+ try {
+ Thread.sleep( cassandraFig.getLockManagerInitInterval() );
+ } catch (InterruptedException ignored) {}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index 431122e..f27e2d7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -17,6 +17,7 @@
*/
package org.apache.usergrid.persistence.collection.exception;
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -56,6 +57,16 @@ public class CollectionRuntimeException extends RuntimeException {
this.applicationScope = scope;
}
+ public boolean isMissingKeyspace() {
+ if ( getCause() instanceof BadRequestException ) {
+ BadRequestException bre = (BadRequestException)getCause();
+ String msg = bre.getMessage();
+ if ( msg.contains("Keyspace") && msg.contains( "does not exist" ) ) {
+ return true;
+ }
+ }
+ return false;
+ }
public ApplicationScope getApplicationScope() {
return applicationScope;
[10/26] usergrid git commit: Merge branch 'release-2.1.1' into
usegrid-1283-mgmt-app-init
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' into usegrid-1283-mgmt-app-init
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8ebc28df
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8ebc28df
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8ebc28df
Branch: refs/heads/release-2.1.1
Commit: 8ebc28dfd8e0d1abe4edec8a3941175238400ea4
Parents: 12743f3 aae8fdf
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 24 10:01:25 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 24 10:01:25 2016 -0400
----------------------------------------------------------------------
.../applications/ApplicationResourceIT.java | 5 +--
.../usergrid/security/shiro/ShiroCache.java | 44 +++++++++++++++++++-
2 files changed, 44 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[24/26] usergrid git commit: Additional test.
Posted by mr...@apache.org.
Additional test.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/317714bc
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/317714bc
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/317714bc
Branch: refs/heads/release-2.1.1
Commit: 317714bc4e5be27a220420c4df4ab281309ba086
Parents: e9228e1
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 3 08:30:42 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 3 08:30:42 2016 -0400
----------------------------------------------------------------------
.../queries/SelectMappingsQueryTest.java | 38 ++++++++++++++++++--
1 file changed, 36 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/317714bc/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index 4a291b6..fd33c15 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -103,7 +103,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
* Field named testProp can be over-written by field named TESTPROP.
*/
@Test
- public void testFieldOverride() throws Exception {
+ public void testFieldOverride1() throws Exception {
String collectionName = "things";
@@ -119,7 +119,7 @@ public class SelectMappingsQueryTest extends QueryTestBase {
app().collection( collectionName ).post( entity );
refreshIndex();
- // testProp and TESTPROP should now have newValue
+ // testProp and TESTPROP should new be queryable by new value
QueryParameters params = new QueryParameters()
.setQuery( "select * where testProp='" + newValue + "'" );
@@ -131,4 +131,38 @@ public class SelectMappingsQueryTest extends QueryTestBase {
things = app().collection( "things" ).get( params );
assertEquals( 1, things.getNumOfEntities() );
}
+
+ /**
+ * Field named testProp can be over-written by field named TESTPROP.
+ */
+ @Test
+ public void testFieldOverride2() throws Exception {
+
+ String collectionName = "things";
+
+ // create entity with TESTPROP=value
+ String value = RandomStringUtils.randomAlphabetic( 20 );
+ Entity entity = new Entity().withProp( "TESTPROP", value );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // override with testProp=newValue
+ String newValue = RandomStringUtils.randomAlphabetic( 20 );
+ entity = new Entity().withProp( "testProp", newValue );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // testProp and TESTPROP should new be queryable by new value
+
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp='" + newValue + "'" );
+ Collection things = this.app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+
+ params = new QueryParameters()
+ .setQuery( "select * where TESTPROP='" + newValue + "'" );
+ things = app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+ }
+
}
[26/26] usergrid git commit: Merge commit 'refs/pull/532/head' of
github.com:apache/usergrid into release-2.1.1
Posted by mr...@apache.org.
Merge commit 'refs/pull/532/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/a1cb1f5f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/a1cb1f5f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/a1cb1f5f
Branch: refs/heads/release-2.1.1
Commit: a1cb1f5f45352a3dc7e7c0df91bb037e027d0ddd
Parents: 8976b41 ed4e67c
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 3 07:42:43 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 3 07:42:43 2016 -0700
----------------------------------------------------------------------
.../usergrid/security/shiro/ShiroCache.java | 80 ++++++++++++--------
1 file changed, 49 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
[19/26] usergrid git commit: Move LockManager setup into normal setup
regime, don't cache EntityManagers with null application names,
and add logging to help debug issue USERGRID-1283.
Posted by mr...@apache.org.
Move LockManager setup into normal setup regime, don't cache EntityManagers with null application names, and add logging to help debug issue USERGRID-1283.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/59c538aa
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/59c538aa
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/59c538aa
Branch: refs/heads/release-2.1.1
Commit: 59c538aa8a6becf90573c815e68347036f1aca44
Parents: 61a35a0
Author: Dave Johnson <sn...@apache.org>
Authored: Tue May 31 09:34:48 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Tue May 31 09:34:48 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 13 +++++-
.../apache/usergrid/locking/LockManager.java | 5 +++
.../cassandra/AstyanaxLockManagerImpl.java | 42 ++++++--------------
.../locking/noop/NoOpLockManagerImpl.java | 5 +++
.../usergrid/services/ServiceManager.java | 8 ++++
.../services/ServiceManagerFactory.java | 12 ++++++
6 files changed, 54 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index ee28765..622944b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -33,6 +33,7 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
import org.apache.usergrid.exception.ConflictException;
+import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
@@ -107,6 +108,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final ConnectionService connectionService;
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
+ private final LockManager lockManager;
public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
@@ -127,6 +129,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.collectionService = injector.getInstance( CollectionService.class );
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
+ this.lockManager = injector.getInstance( LockManager.class );
Properties properties = cassandraService.getProperties();
@@ -166,7 +169,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
// the management app is a special case
-
if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
if ( app != null ) {
@@ -179,6 +181,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
+ // missing keyspace means we have not done bootstrap yet
final boolean missingKeyspace;
if ( throwable instanceof CollectionRuntimeException ) {
CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
@@ -187,6 +190,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
missingKeyspace = false;
}
+ // work around for https://issues.apache.org/jira/browse/USERGRID-1291
+ // throw exception so that we do not cache
+ // TODO: determine how application name can intermittently be null
+ if ( app != null && app.getName() == null ) {
+ throw new RuntimeException( "Name is null for application " + appId, throwable );
+ }
+
if ( app == null && !missingKeyspace ) {
throw new RuntimeException( "Error getting application " + appId, throwable );
@@ -645,6 +655,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void setup() throws Exception {
getSetup().initSchema();
+ lockManager.setup();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java b/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
index a2d37ab..4ed41d8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/LockManager.java
@@ -38,4 +38,9 @@ public interface LockManager {
* @throws UGLockException if the lock cannot be acquired
*/
public Lock createLock( final UUID applicationId, final String... path );
+
+ /**
+ * Setup lock persistence mechanism.
+ */
+ public void setup();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 6acce47..90b9d57 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -50,6 +50,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
private final CassandraFig cassandraFig;
+ private final CassandraCluster cassandraCluster;
private Keyspace keyspace;
private ColumnFamily columnFamily;
private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute
@@ -60,38 +61,19 @@ public class AstyanaxLockManagerImpl implements LockManager {
CassandraCluster cassandraCluster ) throws ConnectionException {
this.cassandraFig = cassandraFig;
+ this.cassandraCluster = cassandraCluster;
+ }
- // hold up construction until we can create the column family
- int maxRetries = cassandraFig.getLockManagerInitRetries();
- int retries = 0;
- boolean famReady = false;
- Set<Class> seenBefore = new HashSet<>(10);
- while ( !famReady && retries++ < maxRetries ) {
- try {
- keyspace = cassandraCluster.getLocksKeyspace();
- createLocksKeyspace();
- columnFamily = createLocksColumnFamily();
- famReady = true;
-
- } catch ( Throwable t ) {
- final String msg;
- if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException) {
- msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName() + ")";
- } else {
- msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries=" + retries;
- }
- if ( !seenBefore.contains( t.getClass() ) ) {
- logger.error( msg, t );
- } else {
- logger.error( msg );
- }
- seenBefore.add( t.getClass() );
- try {
- Thread.sleep( cassandraFig.getLockManagerInitInterval() );
- } catch (InterruptedException ignored) {}
- }
- }
+ @Override
+ public void setup() {
+ try {
+ keyspace = cassandraCluster.getLocksKeyspace();
+ createLocksKeyspace();
+ columnFamily = createLocksColumnFamily();
+ } catch (ConnectionException e) {
+ throw new RuntimeException( "Error setting up locks keyspace and column family", e );
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
index 6174890..ff14031 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/noop/NoOpLockManagerImpl.java
@@ -38,4 +38,9 @@ public class NoOpLockManagerImpl implements LockManager {
public Lock createLock( UUID applicationId, String... path ) {
return new NoOpLockImpl();
}
+
+ @Override
+ public void setup() {
+ // no op
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index a9892f5..04e00e0 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -96,6 +96,14 @@ public class ServiceManager {
this.qm = qm;
this.properties = properties;
+ // additional logging to help debug https://issues.apache.org/jira/browse/USERGRID-1291
+ if ( em == null ) {
+ logger.error("EntityManager is null");
+ }
+ if ( qm == null ) {
+ logger.error("QueueManager is null");
+ }
+
if ( em != null ) {
try {
application = em.getApplication();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/59c538aa/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
index 5274336..2425b95 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManagerFactory.java
@@ -23,6 +23,8 @@ import java.util.UUID;
import com.google.inject.Injector;
import org.apache.usergrid.locking.Lock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
@@ -35,6 +37,7 @@ import org.apache.usergrid.persistence.EntityManagerFactory;
public class ServiceManagerFactory implements ApplicationContextAware {
+ private static final Logger logger = LoggerFactory.getLogger( ServiceManagerFactory.class );
private ApplicationContext applicationContext;
@@ -59,6 +62,15 @@ public class ServiceManagerFactory implements ApplicationContextAware {
public ServiceManager getServiceManager( UUID applicationId ) {
+
+ // additional logging to help debug https://issues.apache.org/jira/browse/USERGRID-1291
+ if ( emf == null ) {
+ logger.error("EntityManagerFactory is null");
+ }
+ if ( qmf == null ) {
+ logger.error("QueueManagerFactory is null");
+ }
+
EntityManager em = null;
if ( emf != null ) {
em = emf.getEntityManager( applicationId );
[14/26] usergrid git commit: Revert all changes to CpEntityManager
Posted by mr...@apache.org.
Revert all changes to CpEntityManager
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/b0fba682
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/b0fba682
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/b0fba682
Branch: refs/heads/release-2.1.1
Commit: b0fba682d652195f56f6e49b09b488ca677780a6
Parents: e299024
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:28:05 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:28:05 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/b0fba682/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 13b5d1f..68f5d71 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -243,16 +243,16 @@ public class CpEntityManager implements EntityManager {
* @param applicationId
*/
public CpEntityManager( final CassandraService cass,
- final CounterUtils counterUtils,
- final AsyncEventService indexService,
- final ManagerCache managerCache,
- final MetricsFactory metricsFactory,
- final EntityManagerFig entityManagerFig,
- final GraphManagerFactory graphManagerFactory,
- final CollectionService collectionService,
- final ConnectionService connectionService,
- final IndexSchemaCacheFactory indexSchemaCacheFactory,
- final UUID applicationId ) {
+ final CounterUtils counterUtils,
+ final AsyncEventService indexService,
+ final ManagerCache managerCache,
+ final MetricsFactory metricsFactory,
+ final EntityManagerFig entityManagerFig,
+ final GraphManagerFactory graphManagerFactory,
+ final CollectionService collectionService,
+ final ConnectionService connectionService,
+ final IndexSchemaCacheFactory indexSchemaCacheFactory,
+ final UUID applicationId ) {
this.entityManagerFig = entityManagerFig;
[11/26] usergrid git commit: Merge branch 'release-2.1.1' into
usergrid-1283-mgmt-app-init
Posted by mr...@apache.org.
Merge branch 'release-2.1.1' into usergrid-1283-mgmt-app-init
Conflicts:
stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/6daba1d7
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/6daba1d7
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/6daba1d7
Branch: refs/heads/release-2.1.1
Commit: 6daba1d7d3dd9ce65644229af87f0440cee53117
Parents: 8ebc28d 0970e1d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:08:06 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:08:06 2016 -0400
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 5 --
.../corepersistence/CpEntityManager.java | 3 -
.../corepersistence/CpEntityManagerFactory.java | 72 +++++++++-----------
.../usergrid/services/ServiceManager.java | 27 ++++----
4 files changed, 46 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6daba1d7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6daba1d7/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 924dd10,91a936d..f92fc9e
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@@ -141,66 -142,12 +141,79 @@@ public class CpEntityManagerFactory imp
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
- // this line always needs to be last due to the temporary circular dependency until spring is removed
-
- this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
- getManagementEntityManager() );
-
- //this line always needs to be last due to the temporary cicular dependency until spring is removed
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt(
+ cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ }
- this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
- getManagementEntityManager() );
+ entityManagers = CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+ public EntityManager load( UUID appId ) { // no checked exception
+
- // get entity manager and ensure it can get its own application
-
++ // create new entity manager and pre-fetch its application
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Exception exception = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Exception e) {
+ exception = e;
+ }
+
+ // the management app is a special case
+
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
- if ( app != null && entityManager != null ) {
-
++ if ( app != null ) {
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
- } else if ( entityManager == null && managementAppEntityManager != null ) {
-
++ } else if ( managementAppEntityManager != null ) {
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
-
- } else {
-
- // fetch failed and we have nothing cached, we must be bootstrapping
- logger.info("managementAppEntityManager is null, bootstrapping in progress");
+ }
++ }
- } else { // not the management app, so blow up if app cannot be fetched
-
- if (app == null) {
- throw new RuntimeException( "Error getting application " + appId, exception );
- }
++ if (app == null) {
++ throw new RuntimeException( "Error getting application " + appId, exception );
+ }
+
+ return entityManager;
+ }
+ });
++
++ // hold up construction until we can access the management app
++ int maxRetries = 1000;
++ int retries = 0;
++ boolean managementAppFound = false;
++ Set<Class> seenBefore = new HashSet<>(100);
++ while ( !managementAppFound && retries++ < maxRetries ) {
++ try {
++ getEntityManager( getManagementAppId() ).getApplication();
++ managementAppFound = true;
++
++ } catch ( Throwable t ) {
++ if ( seenBefore.contains( t.getClass() )) { // don't log full stack trace if we've seen same before
++ logger.error("Error {} getting management app on try {}", t.getClass().getSimpleName(), retries);
++ } else {
++ logger.error("Error getting management app on try {}", t.getClass().getSimpleName(), t);
++ }
++ }
++ }
++
++ if ( !managementAppFound ) {
++ // exception here will prevent WAR from being deployed
++ throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
++ }
++
++ // this line always needs to be last due to the temporary circular dependency until spring is removed
++ applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class)
++ .getInstance( getManagementEntityManager() );
}
@@@ -255,7 -187,7 +252,6 @@@
}
--
@Override
public EntityManager getEntityManager(UUID applicationId) {
try {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/6daba1d7/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --cc stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index 993bf35,a9892f5..e5ef407
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@@ -17,32 -17,32 +17,29 @@@
package org.apache.usergrid.services;
--import java.lang.reflect.Modifier;
--import java.util.*;
--import java.util.concurrent.ExecutionException;
--import java.util.concurrent.TimeUnit;
--
--import org.slf4j.Logger;
--import org.slf4j.LoggerFactory;
--import org.springframework.context.ApplicationContext;
++import com.google.common.cache.CacheBuilder;
++import com.google.common.cache.CacheLoader;
++import com.google.common.cache.LoadingCache;
++import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.batch.service.SchedulerService;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.mq.QueueManager;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
--import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.services.ServiceParameter.IdParameter;
import org.apache.usergrid.services.applications.ApplicationsService;
import org.apache.usergrid.services.exceptions.UndefinedServiceEntityTypeException;
import org.apache.usergrid.utils.ListUtils;
++import org.slf4j.Logger;
++import org.slf4j.LoggerFactory;
++import org.springframework.context.ApplicationContext;
--import org.apache.commons.lang.StringUtils;
--
--import com.google.common.cache.CacheBuilder;
--import com.google.common.cache.CacheLoader;
--import com.google.common.cache.LoadingCache;
++import java.lang.reflect.Modifier;
++import java.util.*;
++import java.util.concurrent.ExecutionException;
++import java.util.concurrent.TimeUnit;
import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
import static org.apache.usergrid.utils.InflectionUtils.pluralize;
[21/26] usergrid git commit: Ensure that field names are treated
consistently and in a case-insensitive way.
Posted by mr...@apache.org.
Ensure that field names are treated consistently and in a case-insensitive way.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5452d68c
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5452d68c
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5452d68c
Branch: refs/heads/release-2.1.1
Commit: 5452d68c29bb59ec8794639cdc8d6d819792e8bb
Parents: 151abf7
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Jun 1 09:56:01 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Jun 1 09:56:01 2016 -0400
----------------------------------------------------------------------
.../read/search/CandidateEntityFilter.java | 13 +++
.../model/field/value/EntityObject.java | 15 ++-
.../queries/SelectMappingsQueryTest.java | 105 +++++++++++++++++++
3 files changed, 128 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5452d68c/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 4aa6c8d..7770436 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -161,6 +161,13 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
}
+ /**
+ * Sets field in result map with support for nested fields via recursion.
+ *
+ * @param result The result map of filtered fields
+ * @param parts The parts of the field name (more than one if field is nested)
+ * @param fieldMap Map of fields of the object
+ */
private void nestedFieldSet( Map<String, Field> result, String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
@@ -184,6 +191,12 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
}
+ /**
+ * Check to see if field should be included in filtered result with support for nested fields via recursion.
+ *
+ * @param parts The parts of the field name (more than one if field is nested)
+ * @param fieldMap Map of fields of the object
+ */
private boolean nestedFieldCheck( String[] parts, Map<String, Field> fieldMap) {
if ( parts.length > 0 ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5452d68c/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
index db44e87..a157029 100644
--- a/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
+++ b/stack/corepersistence/model/src/main/java/org/apache/usergrid/persistence/model/field/value/EntityObject.java
@@ -19,10 +19,7 @@
package org.apache.usergrid.persistence.model.field.value;
import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.*;
import org.apache.usergrid.persistence.model.field.Field;
@@ -41,11 +38,19 @@ public class EntityObject implements Serializable {
@JsonIgnore
private long size;
+ // field names are treated in case-insensitive way by design
+ static class CaseInsensitiveComparator implements Comparator<String> {
+ public int compare(String o1, String o2) {
+ return o1.compareToIgnoreCase(o2);
+ }
+ }
+ public static final CaseInsensitiveComparator INSTANCE = new CaseInsensitiveComparator();
+
/**
* Fields the users can set
*/
@JsonTypeInfo( use=JsonTypeInfo.Id.CLASS, include=JsonTypeInfo.As.PROPERTY, property="@class" )
- private final Map<String, Field> fields = new HashMap<String, Field>();
+ private Map<String, Field> fields = new TreeMap<String, Field>(INSTANCE);
/**
* Add the field, return the old one if it existed
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5452d68c/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
new file mode 100644
index 0000000..eb6aeee
--- /dev/null
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.usergrid.rest.applications.queries;
+
+import org.apache.usergrid.rest.test.resource.model.Collection;
+import org.apache.usergrid.rest.test.resource.model.Entity;
+import org.apache.usergrid.rest.test.resource.model.QueryParameters;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+
+public class SelectMappingsQueryTest extends QueryTestBase {
+ private static final Logger logger = LoggerFactory.getLogger(OrderByTest.class);
+
+
+ @Test
+ public void testNestedSelectFieldNames() throws Exception {
+
+ generateTestEntities(20, "things");
+
+ QueryParameters params = new QueryParameters()
+ .setQuery("select actor.displayName,sometestprop where sometestprop = 'testprop'");
+ Collection things = this.app().collection("things").get(params);
+ assertEquals( 10, things.getNumOfEntities() );
+
+ Iterator<Entity> iter = things.iterator();
+ while ( iter.hasNext() ) {
+
+ Entity entity = iter.next();
+ assertEquals( 5, entity.getDynamicProperties().size() );
+
+ assertNotNull( entity.getDynamicProperties().get("uuid") );
+ assertNotNull( entity.getDynamicProperties().get("type") );
+ assertNotNull( entity.getDynamicProperties().get("metadata") );
+ assertNotNull( entity.getDynamicProperties().get("sometestprop") );
+
+ Map<String, Object> actor = (Map<String, Object>)entity.getDynamicProperties().get("actor");
+ assertNotNull( actor );
+ assertNotNull( actor.get("displayName") );
+
+ }
+ }
+
+
+ /**
+ * Shows that field names are case-insensitive.
+ * If you define two fields with same name but different cases, behavior is undefined.
+ */
+ @Test
+ public void testFieldNameCaseSensitivity() throws Exception {
+
+ int numberOfEntities = 10;
+ String collectionName = "things";
+
+ Entity[] entities = new Entity[numberOfEntities];
+ Entity props = new Entity();
+
+ for (int i = 0; i < numberOfEntities; i++) {
+ props.put("testProp", "a");
+ props.put("testprop", "b");
+ entities[i] = app().collection(collectionName).post(props);
+ }
+ refreshIndex();
+
+ {
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp = 'b'" );
+ Collection things = this.app().collection( "things" ).get( params );
+
+ // if field names were case sensitive, this would fail
+ assertEquals( numberOfEntities, things.getNumOfEntities() );
+ }
+
+ {
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testprop='b'" );
+ Collection things = this.app().collection( "things" ).get( params );
+
+ assertEquals( numberOfEntities, things.getNumOfEntities() );
+ }
+
+ }
+
+}
[22/26] usergrid git commit: Tests to illustrate case-insensitive
handling of Entity field names.
Posted by mr...@apache.org.
Tests to illustrate case-insensitive handling of Entity field names.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e9228e11
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e9228e11
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e9228e11
Branch: refs/heads/release-2.1.1
Commit: e9228e111f338aabf34b334e2661e1966116f23a
Parents: 5452d68
Author: Dave Johnson <sn...@apache.org>
Authored: Thu Jun 2 15:33:28 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu Jun 2 15:33:28 2016 -0400
----------------------------------------------------------------------
.../queries/SelectMappingsQueryTest.java | 79 +++++++++++++-------
1 file changed, 54 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e9228e11/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
index eb6aeee..4a291b6 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/applications/queries/SelectMappingsQueryTest.java
@@ -16,6 +16,7 @@
*/
package org.apache.usergrid.rest.applications.queries;
+import org.apache.commons.lang.RandomStringUtils;
import org.apache.usergrid.rest.test.resource.model.Collection;
import org.apache.usergrid.rest.test.resource.model.Entity;
import org.apache.usergrid.rest.test.resource.model.QueryParameters;
@@ -34,6 +35,9 @@ public class SelectMappingsQueryTest extends QueryTestBase {
private static final Logger logger = LoggerFactory.getLogger(OrderByTest.class);
+ /**
+ * Select field mappings may include nested entity fields.
+ */
@Test
public void testNestedSelectFieldNames() throws Exception {
@@ -64,42 +68,67 @@ public class SelectMappingsQueryTest extends QueryTestBase {
/**
- * Shows that field names are case-insensitive.
- * If you define two fields with same name but different cases, behavior is undefined.
+ * When entity posted with two duplicate names with different cases, last one wins.
*/
@Test
- public void testFieldNameCaseSensitivity() throws Exception {
+ public void testMixedCaseDupField() throws Exception {
- int numberOfEntities = 10;
String collectionName = "things";
- Entity[] entities = new Entity[numberOfEntities];
- Entity props = new Entity();
+ String value = RandomStringUtils.randomAlphabetic( 20 );
+ String otherValue = RandomStringUtils.randomAlphabetic( 20 );
- for (int i = 0; i < numberOfEntities; i++) {
- props.put("testProp", "a");
- props.put("testprop", "b");
- entities[i] = app().collection(collectionName).post(props);
- }
+ // create entity with testProp=value
+ Entity entity = new Entity()
+ .withProp( "testProp", value )
+ .withProp( "TESTPROP", otherValue);
+ app().collection( collectionName ).post( entity );
refreshIndex();
- {
- QueryParameters params = new QueryParameters()
- .setQuery( "select * where testProp = 'b'" );
- Collection things = this.app().collection( "things" ).get( params );
+ // testProp and TESTPROP should now have otherValue
- // if field names were case sensitive, this would fail
- assertEquals( numberOfEntities, things.getNumOfEntities() );
- }
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp='" + otherValue + "'" );
+ Collection things = this.app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+
+ params = new QueryParameters()
+ .setQuery( "select * where TESTPROP='" + otherValue + "'" );
+ things = app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+ }
- {
- QueryParameters params = new QueryParameters()
- .setQuery( "select * where testprop='b'" );
- Collection things = this.app().collection( "things" ).get( params );
- assertEquals( numberOfEntities, things.getNumOfEntities() );
- }
+ /**
+ * Field named testProp can be over-written by field named TESTPROP.
+ */
+ @Test
+ public void testFieldOverride() throws Exception {
- }
+ String collectionName = "things";
+ // create entity with testProp=value
+ String value = RandomStringUtils.randomAlphabetic( 20 );
+ Entity entity = new Entity().withProp( "testProp", value );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // override with TESTPROP=newValue
+ String newValue = RandomStringUtils.randomAlphabetic( 20 );
+ entity = new Entity().withProp( "TESTPROP", newValue );
+ app().collection( collectionName ).post( entity );
+ refreshIndex();
+
+ // testProp and TESTPROP should now have newValue
+
+ QueryParameters params = new QueryParameters()
+ .setQuery( "select * where testProp='" + newValue + "'" );
+ Collection things = this.app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+
+ params = new QueryParameters()
+ .setQuery( "select * where TESTPROP='" + newValue + "'" );
+ things = app().collection( "things" ).get( params );
+ assertEquals( 1, things.getNumOfEntities() );
+ }
}
[05/26] usergrid git commit: Two fields.
Posted by mr...@apache.org.
Two fields.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3eec8e55
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3eec8e55
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3eec8e55
Branch: refs/heads/release-2.1.1
Commit: 3eec8e55eb57b889be1f2d0d13b44d20a98879dd
Parents: 596e0fc
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 18:58:17 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 18:58:17 2016 -0400
----------------------------------------------------------------------
.../src/test/java/org/apache/usergrid/persistence/IndexIT.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3eec8e55/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
index 7e38f17..3b9f95a 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/IndexIT.java
@@ -476,6 +476,7 @@ public class IndexIT extends AbstractCoreIT {
put("data", new HashMap() {{
put("xfactor", 5.1);
put("rando", "bar");
+ put("mondo", "2000");
}});
}};
em.create("names", entity1);
@@ -486,6 +487,7 @@ public class IndexIT extends AbstractCoreIT {
put("data", new HashMap() {{
put("xfactor", 5.1);
put("rando", "bar");
+ put("mondo", "2001");
}});
}};
em.create("names", entity2);
@@ -518,7 +520,7 @@ public class IndexIT extends AbstractCoreIT {
{
// query for only one bogus field name should return empty entities
- Query query = Query.fromQL( "select data.rando where status = 'pickled'" );
+ Query query = Query.fromQL( "select data.rando,data.mondo where status = 'pickled'" );
Results r = em.searchCollection( em.getApplicationRef(), "names", query );
assertTrue( r.getEntities() != null && r.getEntities().size() == 2 );
@@ -526,6 +528,7 @@ public class IndexIT extends AbstractCoreIT {
assertNotNull( first.getProperty("data") );
assertEquals( ((Map<String, Object>)first.getProperty("data")).get("rando"), "bar" );
+ assertEquals( ((Map<String, Object>)first.getProperty("data")).get("mondo"), "2001" );
assertTrue( first.getDynamicProperties().size() == 2 );
}
[13/26] usergrid git commit: Revert all changes to CpEntityManager
Posted by mr...@apache.org.
Revert all changes to CpEntityManager
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/e299024f
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/e299024f
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/e299024f
Branch: refs/heads/release-2.1.1
Commit: e299024ff75caaff3111c9e762671fb91166d50c
Parents: 7d2de99
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:26:27 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:26:27 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 27 +++++++++++++++-----
1 file changed, 20 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/e299024f/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index acdfa0f..13b5d1f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -36,7 +36,6 @@ import java.util.TreeSet;
import java.util.UUID;
import java.util.stream.Collectors;
-import org.apache.usergrid.persistence.*;
import org.apache.usergrid.persistence.collection.EntitySet;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -49,7 +48,24 @@ import org.apache.usergrid.corepersistence.service.CollectionService;
import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.AggregateCounter;
+import org.apache.usergrid.persistence.AggregateCounterSet;
+import org.apache.usergrid.persistence.CollectionRef;
+import org.apache.usergrid.persistence.ConnectedEntityRef;
+import org.apache.usergrid.persistence.ConnectionRef;
+import org.apache.usergrid.persistence.Entity;
+import org.apache.usergrid.persistence.EntityFactory;
+import org.apache.usergrid.persistence.EntityManager;
+import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.IndexBucketLocator;
+import org.apache.usergrid.persistence.Query;
import org.apache.usergrid.persistence.Query.Level;
+import org.apache.usergrid.persistence.RelationManager;
+import org.apache.usergrid.persistence.Results;
+import org.apache.usergrid.persistence.Schema;
+import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.SimpleRoleRef;
+import org.apache.usergrid.persistence.TypedEntity;
import org.apache.usergrid.persistence.cassandra.ApplicationCF;
import org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils;
import org.apache.usergrid.persistence.cassandra.CassandraService;
@@ -214,9 +230,6 @@ public class CpEntityManager implements EntityManager {
private EntityCollectionManager ecm;
- private CpEntityManagerFactory emf;
-
-
// /** Short-term cache to keep us from reloading same Entity during single request. */
// private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
@@ -229,8 +242,7 @@ public class CpEntityManager implements EntityManager {
* @param metricsFactory
* @param applicationId
*/
- public CpEntityManager(final CpEntityManagerFactory emf,
- final CassandraService cass,
+ public CpEntityManager( final CassandraService cass,
final CounterUtils counterUtils,
final AsyncEventService indexService,
final ManagerCache managerCache,
@@ -242,7 +254,6 @@ public class CpEntityManager implements EntityManager {
final IndexSchemaCacheFactory indexSchemaCacheFactory,
final UUID applicationId ) {
- this.emf = emf;
this.entityManagerFig = entityManagerFig;
Preconditions.checkNotNull( cass, "cass must not be null" );
@@ -259,6 +270,8 @@ public class CpEntityManager implements EntityManager {
this.connectionService = connectionService;
this.collectionService = collectionService;
+
+
this.managerCache = managerCache;
this.applicationId = applicationId;
this.indexService = indexService;
[23/26] usergrid git commit: Remove unneeded properties,
log when mgmt app lookup fails, minor refactoring
Posted by mr...@apache.org.
Remove unneeded properties, log when mgmt app lookup fails, minor refactoring
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/4b8e5460
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/4b8e5460
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/4b8e5460
Branch: refs/heads/release-2.1.1
Commit: 4b8e5460cb93fb65ee54282e93887668bf7d6f8a
Parents: 59c538a
Author: Dave Johnson <sn...@apache.org>
Authored: Fri Jun 3 07:31:46 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Fri Jun 3 07:31:46 2016 -0400
----------------------------------------------------------------------
.../src/main/resources/usergrid-default.properties | 7 -------
.../corepersistence/CpEntityManagerFactory.java | 14 +++++++-------
.../exception/CollectionRuntimeException.java | 2 +-
3 files changed, 8 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b8e5460/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index 1e22d6a..0fc31ef 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -195,13 +195,6 @@ cassandra.lock.writecl=LOCAL_QUORUM
#
#cassandra.useSocketKeepalive=false
-# Number of times to retry creation of lock keyspace and column family
-cassandra.lock.init.retries = 100;
-
-# Interval between lock keyspace creation attempts (in milliseconds)
-cassandra.lock.init.interval = 1000;
-
-
##################### Async Threadpool Settings #####################
# Set the number of threads available in the Rx Async Thread Pool
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b8e5460/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 622944b..18df205 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -69,7 +69,6 @@ import java.util.*;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import static org.apache.usergrid.persistence.Schema.initLock;
/**
@@ -178,26 +177,27 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
} else if ( managementAppEntityManager != null ) {
// failed to fetch management app, use cached one
entityManager = managementAppEntityManager;
+ logger.error("Failed to fetch management app");
}
}
// missing keyspace means we have not done bootstrap yet
- final boolean missingKeyspace;
+ final boolean isBootstrapping;
if ( throwable instanceof CollectionRuntimeException ) {
CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
- missingKeyspace = cre.isMissingKeyspace();
+ isBootstrapping = cre.isBootstrapping();
} else {
- missingKeyspace = false;
+ isBootstrapping = false;
}
// work around for https://issues.apache.org/jira/browse/USERGRID-1291
- // throw exception so that we do not cache
+ // throw exception so that we do not cache
// TODO: determine how application name can intermittently be null
if ( app != null && app.getName() == null ) {
throw new RuntimeException( "Name is null for application " + appId, throwable );
}
- if ( app == null && !missingKeyspace ) {
+ if ( app == null && !isBootstrapping ) {
throw new RuntimeException( "Error getting application " + appId, throwable );
} // else keyspace is missing because setup/bootstrap not done yet
@@ -241,7 +241,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
if ( t instanceof CollectionRuntimeException ) {
CollectionRuntimeException cre = (CollectionRuntimeException)t;
- if ( cre.isMissingKeyspace() ) {
+ if ( cre.isBootstrapping() ) {
// we're bootstrapping, ignore this and continue
bootstrapping = true;
break;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/4b8e5460/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index f27e2d7..8aa2a7a 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -57,7 +57,7 @@ public class CollectionRuntimeException extends RuntimeException {
this.applicationScope = scope;
}
- public boolean isMissingKeyspace() {
+ public boolean isBootstrapping() {
if ( getCause() instanceof BadRequestException ) {
BadRequestException bre = (BadRequestException)getCause();
String msg = bre.getMessage();
[25/26] usergrid git commit: Merge commit 'refs/pull/528/head' of
github.com:apache/usergrid into release-2.1.1
Posted by mr...@apache.org.
Merge commit 'refs/pull/528/head' of github.com:apache/usergrid into release-2.1.1
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/8976b414
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/8976b414
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/8976b414
Branch: refs/heads/release-2.1.1
Commit: 8976b414a7e62fa22f9e894eaf50edfe60c4dd76
Parents: 317714b 4b8e546
Author: Michael Russo <mr...@apigee.com>
Authored: Fri Jun 3 07:42:30 2016 -0700
Committer: Michael Russo <mr...@apigee.com>
Committed: Fri Jun 3 07:42:30 2016 -0700
----------------------------------------------------------------------
.../main/resources/usergrid-default.properties | 3 +-
.../corepersistence/CpEntityManagerFactory.java | 243 ++++++++++++++-----
.../apache/usergrid/locking/LockManager.java | 5 +
.../cassandra/AstyanaxLockManagerImpl.java | 30 ++-
.../locking/noop/NoOpLockManagerImpl.java | 5 +
.../exception/CollectionRuntimeException.java | 11 +
.../core/astyanax/CassandraCluster.java | 24 +-
.../persistence/core/astyanax/CassandraFig.java | 18 +-
.../usergrid/services/ServiceManager.java | 8 +
.../services/ServiceManagerFactory.java | 12 +
10 files changed, 285 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
[02/26] usergrid git commit: Improve cache logic to avoid caching
"bad" entity managers and to ensure that manager app is always available.
Should be no reason to use a retry loop because the system will continue to
call for EntityManagers. Adds new "ent
Posted by mr...@apache.org.
Improve cache logic to avoid caching "bad" entity managers and to ensure that manager app is always available.
Should be no reason to use a retry loop because the system will continue to call for EntityManagers.
Adds new "entity.manager.cache.size" property, which defaults to 100.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5f463412
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5f463412
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5f463412
Branch: refs/heads/release-2.1.1
Commit: 5f463412b661c7b681489d8831fc2c7c7d32938a
Parents: 3a96239
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 18 09:18:21 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 18 09:18:21 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 70 ++++++++++++++++++--
1 file changed, 63 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5f463412/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 91a936d..4028875 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -102,13 +102,14 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private Setup setup = null;
+ EntityManager managementAppEntityManager = null;
+
// cache of already instantiated entity managers
- private LoadingCache<UUID, EntityManager> entityManagers
- = CacheBuilder.newBuilder().maximumSize(100).build(new CacheLoader<UUID, EntityManager>() {
- public EntityManager load(UUID appId) { // no checked exception
- return _getEntityManager(appId);
- }
- });
+ private final String ENTITY_MANAGER_CACHE_SIZE = "entity.manager.cache.size";
+ private final LoadingCache<UUID, EntityManager> entityManagers;
+
+
+
private final ApplicationIdCache applicationIdCache;
//private final IndexSchemaCache indexSchemaCache;
@@ -142,12 +143,67 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.connectionService = injector.getInstance( ConnectionService.class );
this.indexSchemaCacheFactory = injector.getInstance( IndexSchemaCacheFactory.class );
- //this line always needs to be last due to the temporary cicular dependency until spring is removed
+ // this line always needs to be last due to the temporary circular dependency until spring is removed
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
getManagementEntityManager() );
+ int entityManagerCacheSize = 100;
+ try {
+ entityManagerCacheSize = Integer.parseInt(
+ cassandraService.getProperties().getProperty( ENTITY_MANAGER_CACHE_SIZE, "100" ));
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + ENTITY_MANAGER_CACHE_SIZE + " using " + entityManagerCacheSize, e );
+ }
+
+ entityManagers = CacheBuilder.newBuilder()
+ .maximumSize(entityManagerCacheSize)
+ .build(new CacheLoader<UUID, EntityManager>() {
+
+ public EntityManager load( UUID appId ) { // no checked exception
+
+ // get entity manager and ensure it can get its own application
+
+ EntityManager entityManager = _getEntityManager( appId );
+ Application app = null;
+ Exception exception = null;
+ try {
+ app = entityManager.getApplication();
+ } catch (Exception e) {
+ exception = e;
+ }
+
+ // the management app is a special case
+
+ if ( CpNamingUtils.MANAGEMENT_APPLICATION_ID.equals( appId ) ) {
+
+ if ( app != null ) {
+
+ // we successfully fetched up the management app, cache it for a rainy day
+ managementAppEntityManager = entityManager;
+
+ } else if ( managementAppEntityManager != null ) {
+
+ // failed to fetch management app, use cached one
+ entityManager = managementAppEntityManager;
+
+ } else {
+
+ // fetch failed and we have nothing cached, we must be bootstrapping
+ logger.info("managementAppEntityManager is null, bootstrapping in progress");
+ }
+
+ } else { // not the management app, so blow up if app cannot be fetched
+
+ if (app == null) {
+ throw new RuntimeException( "Error getting application " + appId, exception );
+ }
+ }
+
+ return entityManager;
+ }
+ });
}
[12/26] usergrid git commit: Revert all changes to ServiceManager
Posted by mr...@apache.org.
Revert all changes to ServiceManager
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/7d2de990
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/7d2de990
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/7d2de990
Branch: refs/heads/release-2.1.1
Commit: 7d2de990705c0fb8380a03620862815854e9fd73
Parents: 6daba1d
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 10:20:24 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 10:20:24 2016 -0400
----------------------------------------------------------------------
.../usergrid/services/ServiceManager.java | 25 +++++++++++---------
1 file changed, 14 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/7d2de990/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
index e5ef407..a9892f5 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/ServiceManager.java
@@ -17,29 +17,32 @@
package org.apache.usergrid.services;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import org.apache.commons.lang.StringUtils;
+import java.lang.reflect.Modifier;
+import java.util.*;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.context.ApplicationContext;
import org.apache.usergrid.batch.service.SchedulerService;
import org.apache.usergrid.locking.LockManager;
import org.apache.usergrid.mq.QueueManager;
import org.apache.usergrid.persistence.Entity;
import org.apache.usergrid.persistence.EntityManager;
import org.apache.usergrid.persistence.EntityRef;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.entities.Application;
import org.apache.usergrid.services.ServiceParameter.IdParameter;
import org.apache.usergrid.services.applications.ApplicationsService;
import org.apache.usergrid.services.exceptions.UndefinedServiceEntityTypeException;
import org.apache.usergrid.utils.ListUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.context.ApplicationContext;
-import java.lang.reflect.Modifier;
-import java.util.*;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.StringUtils;
+
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
import static org.apache.usergrid.persistence.SimpleEntityRef.ref;
import static org.apache.usergrid.utils.InflectionUtils.pluralize;
[16/26] usergrid git commit: Add retry logic and logging in
AstyanaxLockManagerImpl and CpEntityManagerFactory,
also ASL header in CassandraCluster
Posted by mr...@apache.org.
Add retry logic and logging in AstyanaxLockManagerImpl and CpEntityManagerFactory, also ASL header in CassandraCluster
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/910811d2
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/910811d2
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/910811d2
Branch: refs/heads/release-2.1.1
Commit: 910811d258b359a3c207c1ef34014d1ba28afebe
Parents: ba10e7f
Author: Dave Johnson <sn...@apache.org>
Authored: Wed May 25 13:56:51 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed May 25 13:56:51 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 8 ++---
.../cassandra/AstyanaxLockManagerImpl.java | 36 +++++++++++++-------
.../core/astyanax/CassandraCluster.java | 24 ++++++++++---
3 files changed, 47 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/910811d2/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index d2417be..84872aa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -174,17 +174,17 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
int maxRetries = 1000;
int retries = 0;
boolean managementAppFound = false;
- Set<Class> seenBefore = new HashSet<>(100);
while ( !managementAppFound && retries++ < maxRetries ) {
try {
getEntityManager( getManagementAppId() ).getApplication();
managementAppFound = true;
} catch ( Throwable t ) {
- if ( seenBefore.contains( t.getClass() )) { // don't log full stack trace if we've seen same before
- logger.error("Error {} getting management app on try {}", t.getClass().getSimpleName(), retries);
+ String msg = "Error " + t.getClass() + " getting management app on try " + retries;
+ if ( logger.isDebugEnabled() ) {
+ logger.error( msg, t);
} else {
- logger.error("Error getting management app on try {}", t.getClass().getSimpleName(), t);
+ logger.error(msg);
}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/910811d2/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index a69aee2..49ff52e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -37,9 +37,7 @@ import org.apache.usergrid.persistence.core.astyanax.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.TimeUnit;
@Singleton
@@ -50,8 +48,8 @@ public class AstyanaxLockManagerImpl implements LockManager {
private final CassandraFig cassandraFig;
- private final Keyspace keyspace;
- private final ColumnFamily columnFamily;
+ private Keyspace keyspace;
+ private ColumnFamily columnFamily;
private static final int MINIMUM_LOCK_EXPIRATION = 60000; // 1 minute
@Inject
@@ -59,14 +57,28 @@ public class AstyanaxLockManagerImpl implements LockManager {
CassandraCluster cassandraCluster ) throws ConnectionException {
this.cassandraFig = cassandraFig;
- this.keyspace = cassandraCluster.getLocksKeyspace();
-
- createLocksKeyspace();
-
- this.columnFamily = createLocksColumnFamily();
-
-
+ // hold up construction until we can create the column family
+ int maxRetries = 1000;
+ int retries = 0;
+ boolean famReady = false;
+ while ( !famReady && retries++ < maxRetries ) {
+ try {
+ keyspace = cassandraCluster.getLocksKeyspace();
+ createLocksKeyspace();
+ columnFamily = createLocksColumnFamily();
+ famReady = true;
+
+ } catch ( Throwable t ) {
+ String msg = "Error " + t.getClass().getSimpleName() + " creating locks keyspace try " + retries;
+ if ( logger.isDebugEnabled() ) {
+ logger.error( msg, t );
+ } else {
+ logger.error( msg );
+ }
+ try { Thread.sleep(1000); } catch (InterruptedException ignored) {}
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/910811d2/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
index 0adac8e..6abc143 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/astyanax/CassandraCluster.java
@@ -1,15 +1,29 @@
-package org.apache.usergrid.persistence.core.astyanax;
-
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.usergrid.persistence.core.astyanax;
import com.netflix.astyanax.Keyspace;
-
import java.util.Map;
-
public interface CassandraCluster {
-
Map<String, Keyspace> getKeyspaces();
Keyspace getApplicationKeyspace();