You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by mr...@apache.org on 2016/06/03 14:46:08 UTC
[18/26] usergrid git commit: Made max-retries and intervals
configurable for both lock manager and entity manager factory,
also better error logging.
Made max-retries and intervals configurable for both lock manager and entity manager factory, also better error logging.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/61a35a04
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/61a35a04
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/61a35a04
Branch: refs/heads/release-2.1.1
Commit: 61a35a041598af0e336199343ed6a489454980f6
Parents: fcd00e8
Author: Dave Johnson <sn...@apache.org>
Authored: Thu May 26 11:10:33 2016 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Thu May 26 11:10:33 2016 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 83 ++++++++++++++++----
.../cassandra/AstyanaxLockManagerImpl.java | 17 +++-
.../exception/CollectionRuntimeException.java | 11 +++
3 files changed, 93 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index b6fbc2a..ee28765 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -20,11 +20,10 @@ import com.google.common.base.Optional;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.UncheckedExecutionException;
import com.google.inject.Injector;
import com.google.inject.Key;
import com.google.inject.TypeLiteral;
-import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
-import org.apache.cassandra.exceptions.InvalidRequestException;
import org.apache.commons.lang.StringUtils;
import org.apache.usergrid.corepersistence.asyncevents.AsyncEventService;
import org.apache.usergrid.corepersistence.index.IndexSchemaCacheFactory;
@@ -39,6 +38,7 @@ import org.apache.usergrid.persistence.cassandra.CassandraService;
import org.apache.usergrid.persistence.cassandra.CounterUtils;
import org.apache.usergrid.persistence.cassandra.Setup;
import org.apache.usergrid.persistence.collection.EntityCollectionManager;
+import org.apache.usergrid.persistence.collection.exception.CollectionRuntimeException;
import org.apache.usergrid.persistence.collection.serialization.impl.migration.EntityIdScope;
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.migration.data.MigrationDataProvider;
@@ -68,6 +68,7 @@ import java.util.*;
import static java.lang.String.CASE_INSENSITIVE_ORDER;
import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.initLock;
/**
@@ -107,7 +108,8 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private final GraphManagerFactory graphManagerFactory;
private final IndexSchemaCacheFactory indexSchemaCacheFactory;
- public static final String MANAGEMENT_APP_MAX_RETRIES= "management.app.max.retries";
+ public static final String MANAGEMENT_APP_INIT_MAXRETRIES= "management.app.init.max-retries";
+ public static final String MANAGEMENT_APP_INIT_INTERVAL = "management.app.init.interval";
public CpEntityManagerFactory(
@@ -177,12 +179,19 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
- if ( app == null && throwable != null && throwable.getCause() instanceof BadRequestException) {
- // probably means schema has not been created yet
+ final boolean missingKeyspace;
+ if ( throwable instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException) throwable;
+ missingKeyspace = cre.isMissingKeyspace();
} else {
- throw new RuntimeException( "Error getting application " + appId, throwable );
+ missingKeyspace = false;
}
+ if ( app == null && !missingKeyspace ) {
+ throw new RuntimeException( "Error getting application " + appId, throwable );
+
+ } // else keyspace is missing because setup/bootstrap not done yet
+
return entityManager;
}
});
@@ -190,34 +199,77 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private void checkManagementApp(Properties properties) {
+
int maxRetries = 100;
try {
- maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_MAX_RETRIES, "100" ));
+ maxRetries = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_MAXRETRIES, "100" ));
+
+ } catch ( Exception e ) {
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_MAXRETRIES + ". Will use " + maxRetries, e );
+ }
+
+ int interval = 1000;
+ try {
+ interval = Integer.parseInt( properties.getProperty( MANAGEMENT_APP_INIT_INTERVAL, "1000" ));
} catch ( Exception e ) {
- logger.error("Error parsing " + MANAGEMENT_APP_MAX_RETRIES + ". Will use " + maxRetries, e );
+ logger.error("Error parsing " + MANAGEMENT_APP_INIT_INTERVAL + ". Will use " + maxRetries, e );
}
// hold up construction until we can access the management app
int retries = 0;
boolean managementAppFound = false;
+ boolean bootstrapping = false;
+ Set<Class> seenBefore = new HashSet<>(10);
while ( !managementAppFound && retries++ < maxRetries ) {
try {
- getEntityManager( getManagementAppId() ).getApplication();
+ // bypass entity manager cache and get managementApp
+ managementApp = _getEntityManager( getManagementAppId() ).getApplication();
managementAppFound = true;
} catch ( Throwable t ) {
- String msg = "Error " + t.getClass() + " getting management app on try " + retries;
- if ( logger.isDebugEnabled() ) {
+
+ if ( t instanceof CollectionRuntimeException ) {
+ CollectionRuntimeException cre = (CollectionRuntimeException)t;
+ if ( cre.isMissingKeyspace() ) {
+ // we're bootstrapping, ignore this and continue
+ bootstrapping = true;
+ break;
+ }
+ }
+ Throwable cause = t;
+
+ // there was an error, be as informative as possible
+ StringBuilder sb = new StringBuilder();
+ sb.append(retries).append(": Error (");
+
+ if ( t instanceof UncheckedExecutionException ) {
+ UncheckedExecutionException uee = (UncheckedExecutionException)t;
+ if ( uee.getCause() instanceof RuntimeException ) {
+ cause = uee.getCause().getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ")
+ .append(uee.getCause().getMessage());
+ } else {
+ cause = uee.getCause();
+ sb.append(cause.getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+ } else {
+ sb.append(t.getCause().getClass().getSimpleName()).append(") ").append(t.getMessage());
+ }
+
+ String msg = sb.toString();
+ if ( !seenBefore.contains( cause.getClass() ) ) {
logger.error( msg, t);
} else {
logger.error(msg);
}
- try { Thread.sleep( 1000 ); } catch (InterruptedException ignored) {}
+ seenBefore.add( cause.getClass() );
+
+ try { Thread.sleep( interval ); } catch (InterruptedException ignored) {}
}
}
- if ( !managementAppFound ) {
+ if ( !managementAppFound && !bootstrapping ) {
// exception here will prevent WAR from being deployed
throw new RuntimeException( "Unable to get management app after " + retries + " retries" );
}
@@ -323,8 +375,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
// Ensure the management application is initialized
initMgmtAppInternal();
- EntityManager managementEm = getEntityManager( getManagementAppId() );
- EntityManager appEm = getEntityManager(applicationId);
+ // Get entity managers by bypassing the entity manager cache because it expects apps to already exist
+ final EntityManager managementEm = _getEntityManager( getManagementAppId() );
+ EntityManager appEm = _getEntityManager(applicationId);
final String appName = buildAppName(organizationName, name);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
index 9bf6694..6acce47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/locking/cassandra/AstyanaxLockManagerImpl.java
@@ -22,6 +22,8 @@ import com.google.inject.Inject;
import com.google.inject.Singleton;
import com.netflix.astyanax.Keyspace;
import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
+import com.netflix.astyanax.connectionpool.exceptions.NoAvailableHostsException;
+import com.netflix.astyanax.connectionpool.exceptions.PoolTimeoutException;
import com.netflix.astyanax.ddl.ColumnFamilyDefinition;
import com.netflix.astyanax.ddl.KeyspaceDefinition;
import com.netflix.astyanax.model.ColumnFamily;
@@ -63,6 +65,7 @@ public class AstyanaxLockManagerImpl implements LockManager {
int maxRetries = cassandraFig.getLockManagerInitRetries();
int retries = 0;
boolean famReady = false;
+ Set<Class> seenBefore = new HashSet<>(10);
while ( !famReady && retries++ < maxRetries ) {
try {
keyspace = cassandraCluster.getLocksKeyspace();
@@ -71,13 +74,21 @@ public class AstyanaxLockManagerImpl implements LockManager {
famReady = true;
} catch ( Throwable t ) {
- String msg = "Error " + t.getClass().getSimpleName() + " creating locks keyspace try " + retries;
- if ( logger.isDebugEnabled() ) {
+ final String msg;
+ if ( t instanceof PoolTimeoutException || t instanceof NoAvailableHostsException) {
+ msg = retries + ": Cannot connect to Cassandra (" + t.getClass().getSimpleName() + ")";
+ } else {
+ msg = retries + ": Error (" + t.getClass().getSimpleName() + ") tries=" + retries;
+ }
+ if ( !seenBefore.contains( t.getClass() ) ) {
logger.error( msg, t );
} else {
logger.error( msg );
}
- try { Thread.sleep( cassandraFig.getLockManagerInitInterval() ); } catch (InterruptedException ignored) {}
+ seenBefore.add( t.getClass() );
+ try {
+ Thread.sleep( cassandraFig.getLockManagerInitInterval() );
+ } catch (InterruptedException ignored) {}
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/61a35a04/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
index 431122e..f27e2d7 100644
--- a/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
+++ b/stack/corepersistence/collection/src/main/java/org/apache/usergrid/persistence/collection/exception/CollectionRuntimeException.java
@@ -17,6 +17,7 @@
*/
package org.apache.usergrid.persistence.collection.exception;
+import com.netflix.astyanax.connectionpool.exceptions.BadRequestException;
import org.apache.usergrid.persistence.collection.MvccEntity;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -56,6 +57,16 @@ public class CollectionRuntimeException extends RuntimeException {
this.applicationScope = scope;
}
+ public boolean isMissingKeyspace() {
+ if ( getCause() instanceof BadRequestException ) {
+ BadRequestException bre = (BadRequestException)getCause();
+ String msg = bre.getMessage();
+ if ( msg.contains("Keyspace") && msg.contains( "does not exist" ) ) {
+ return true;
+ }
+ }
+ return false;
+ }
public ApplicationScope getApplicationScope() {
return applicationScope;