You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2017/08/28 23:23:59 UTC
[13/15] usergrid git commit: 1. Add missing start for delete queue
workers. 2. Add dropping of application local keyspace for testing. 3.
Testing changes.
1. Add missing start for delete queue workers.
2. Add dropping of application local keyspace for testing.
3. Testing changes.
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/3f7afcd8
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/3f7afcd8
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/3f7afcd8
Branch: refs/heads/master
Commit: 3f7afcd8836ed9b589b69523d8bcc507a739d912
Parents: c201f1f
Author: Mike Dunker <md...@google.com>
Authored: Mon Aug 21 17:01:48 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Aug 21 17:01:48 2017 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 3 +--
.../asyncevents/AsyncEventServiceImpl.java | 4 +++
.../index/CollectionSettingsImpl.java | 1 -
.../usergrid/persistence/EntityManager.java | 2 +-
.../persistence/entities/Application.java | 26 ++++++++++++++++++++
.../usergrid/persistence/CoreSchemaManager.java | 16 ++++++++----
.../usergrid/persistence/RebuildIndexTest.java | 24 +++++++++++++-----
.../resources/usergrid-custom-test.properties | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
.../AbstractServiceNotificationIT.java | 4 +--
.../gcm/NotificationsServiceIT.java | 2 +-
.../resources/usergrid-custom-test.properties | 2 +-
12 files changed, 67 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index cdb4fc7..68c4ef0 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2471,10 +2471,9 @@ public class CpEntityManager implements EntityManager {
final Entity entity;
- //this is the fall back, why isn't this writt
if ( entityType == null ) {
return null;
-// throw new EntityNotFoundException( String.format( "Counld not find type for uuid {}", uuid ) );
+// throw new EntityNotFoundException( String.format( "Could not find type for uuid {}", uuid ) );
}
entity = get( new SimpleEntityRef( entityType, uuid ) );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 79a80c0..428772f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -875,6 +875,10 @@ public class AsyncEventServiceImpl implements AsyncEventService {
startWorker(AsyncEventQueueType.UTILITY);
}
+ for (int i = 0; i < deleteCount; i++) {
+ startWorker(AsyncEventQueueType.DELETE);
+ }
+
if( indexQueue instanceof SNSQueueManagerImpl) {
logger.info("Queue manager implementation supports dead letters, start dead letter queue workers.");
for (int i = 0; i < indexDeadCount; i++) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
index 921777a..76f0b4b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/CollectionSettingsImpl.java
@@ -20,7 +20,6 @@ package org.apache.usergrid.corepersistence.index;
import com.google.common.base.Optional;
import com.google.inject.Inject;
-import com.google.inject.Singleton;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.utils.JsonUtils;
import org.slf4j.Logger;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index ae4623d..c9752c3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -743,7 +743,7 @@ public interface EntityManager {
/**
* Add a new index to the application for scale
- * @param suffix unique indentifier for additional index
+ * @param newIndexName unique identifier for additional index
* @param shards number of shards
* @param replicas number of replicas
* @param writeConsistency only "one, quorum, or all"
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
index 790b4d9..0a4360f 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/entities/Application.java
@@ -48,6 +48,32 @@ public class Application extends TypedEntity implements Serializable {
public static final String COLLECTION_ACTIVITIES = "activities";
+ public static final String COLLECTION_EVENTS = "events";
+
+ public static final String COLLECTION_FOLDERS = "folders";
+
+ public static final String COLLECTION_DEVICES = "devices";
+
+ public static final String COLLECTION_NOTIFICATIONS = "notifications";
+
+ public static final String COLLECTION_ROLES = "roles";
+
+ public static boolean isCustomCollectionName(String collectionName) {
+ switch (collectionName.toLowerCase()) {
+ case COLLECTION_USERS:
+ case COLLECTION_GROUPS:
+ case COLLECTION_ASSETS:
+ case COLLECTION_ACTIVITIES:
+ case COLLECTION_EVENTS:
+ case COLLECTION_FOLDERS:
+ case COLLECTION_DEVICES:
+ case COLLECTION_NOTIFICATIONS:
+ case COLLECTION_ROLES:
+ return false;
+ }
+ return true;
+ }
+
@EntityProperty(indexed = true, fulltextIndexed = false, required = true, mutable = false, aliasProperty = true,
basic = true)
protected String name;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
index c064751..1d51c25 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/CoreSchemaManager.java
@@ -18,6 +18,7 @@ package org.apache.usergrid.persistence;
import org.apache.usergrid.locking.LockManager;
+import org.apache.usergrid.persistence.core.CassandraConfig;
import org.apache.usergrid.persistence.core.CassandraFig;
import org.apache.usergrid.persistence.core.datastax.CQLUtils;
import org.apache.usergrid.persistence.core.datastax.DataStaxCluster;
@@ -37,14 +38,14 @@ public class CoreSchemaManager implements SchemaManager {
private static final Logger logger = LoggerFactory.getLogger( CoreSchemaManager.class );
private final Setup setup;
- private final CassandraFig cassandraFig;
+ private final CassandraConfig cassandraConfig;
private final LockManager lockManager;
private final DataStaxCluster dataStaxCluster;
public CoreSchemaManager( final Setup setup, Injector injector ) {
this.setup = setup;
- this.cassandraFig = injector.getInstance( CassandraFig.class );
+ this.cassandraConfig = injector.getInstance( CassandraConfig.class );
this.lockManager = injector.getInstance( LockManager.class );
this.dataStaxCluster = injector.getInstance( DataStaxCluster.class );
}
@@ -80,16 +81,21 @@ public class CoreSchemaManager implements SchemaManager {
@Override
public void destroy() {
- logger.info( "dropping keyspaces" );
try {
+ logger.info( "dropping application keyspace" );
dataStaxCluster.getClusterSession()
- .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraFig.getApplicationKeyspace()));
+ .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationKeyspace()));
+ dataStaxCluster.waitForSchemaAgreement();
+
+ logger.info( "dropping application local keyspace" );
+ dataStaxCluster.getClusterSession()
+ .execute("DROP KEYSPACE "+ CQLUtils.quote(cassandraConfig.getApplicationLocalKeyspace()));
dataStaxCluster.waitForSchemaAgreement();
dataStaxCluster.getClusterSession().close(); // close session so it's meta will get refreshed
}
catch ( Exception e ) {
- logger.error("Error dropping application keyspace: {} error: {}", cassandraFig.getApplicationKeyspace(), e);
+ logger.error("Error dropping application keyspaces: {} error: {}", cassandraConfig.getApplicationKeyspace(), e);
}
logger.info( "keyspaces dropped" );
logger.info( "dropping indices" );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
index d2bff37..a1ff84b 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/RebuildIndexTest.java
@@ -74,7 +74,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
}
- @Test( timeout = 120000 )
+ @Test( timeout = 240000 )
public void rebuildOneCollectionIndex() throws Exception {
logger.info( "Started rebuildOneCollectionIndex()" );
@@ -163,7 +163,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
waitForRebuild( status, reIndexService );
- app.waitForQueueDrainAndRefreshIndex(5000);
+ app.waitForQueueDrainAndRefreshIndex(15000);
// ----------------- test that we can read the catherder collection and not the catshepard
@@ -278,7 +278,7 @@ public class RebuildIndexTest extends AbstractCoreIT {
assertNotNull( status.getJobId(), "JobId is present" );
- logger.info( "Rebuilt index" );
+ logger.info( "Rebuilt index, jobID={}", status.getJobId());
waitForRebuild( status, reIndexService );
@@ -511,14 +511,26 @@ public class RebuildIndexTest extends AbstractCoreIT {
* Wait for the rebuild to occur
*/
private void waitForRebuild( final ReIndexService.ReIndexStatus status, final ReIndexService reIndexService )
- throws InterruptedException {
+ throws InterruptedException, IllegalArgumentException {
+ if (status != null) {
+ logger.info("waitForRebuild: jobID={}", status.getJobId());
+ } else {
+ logger.info("waitForRebuild: error, status = null");
+ throw new IllegalArgumentException("reindexStatus = null");
+ }
while ( true ) {
try {
final ReIndexService.ReIndexStatus updatedStatus = reIndexService.getStatus( status.getJobId() );
- if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
- break;
+ if (updatedStatus == null) {
+ logger.info("waitForRebuild: updated status is null");
+ } else {
+ logger.info("waitForRebuild: status={} numberProcessed={}", updatedStatus.getStatus().toString(), updatedStatus.getNumberProcessed());
+
+ if ( updatedStatus.getStatus() == ReIndexService.Status.COMPLETE ) {
+ break;
+ }
}
}
catch ( IllegalArgumentException iae ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index df253f0..24f3046 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -29,7 +29,7 @@ elasticsearch.management_number_replicas=0
elasticsearch.managment_index=usergrid_core_management
#cassandra.keyspace.application=core_tests_schema
elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
elasticsearch.buffer_timeout=1
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 4b76ca4..ee7b039 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -68,7 +68,7 @@ usergrid.cluster.port=2555
collection.uniquevalues.actors=300
collection.uniquevalues.authoritative.region=us-east
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
# Queueing Test Settings
# Reduce the long polling time for the tests
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
index c035192..9b3dfcd 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/AbstractServiceNotificationIT.java
@@ -49,9 +49,9 @@ public abstract class AbstractServiceNotificationIT extends AbstractServiceIT {
protected Notification notificationWaitForComplete(Notification notification)
throws Exception {
- long timeout = System.currentTimeMillis() + 60000;
+ long timeout = System.currentTimeMillis() + 120000;
while (System.currentTimeMillis() < timeout) {
- app.waitForQueueDrainAndRefreshIndex(200);
+ app.waitForQueueDrainAndRefreshIndex(1000);
notification = app.getEntityManager().get(notification.getUuid(), Notification.class);
if (notification.getFinished() != null) {
return notification;
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
index 8360009..ecc5443 100644
--- a/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/services/notifications/gcm/NotificationsServiceIT.java
@@ -632,7 +632,7 @@ public class NotificationsServiceIT extends AbstractServiceNotificationIT {
// receipts are created and queried, wait a bit longer for this to happen as indexing
- app.waitForQueueDrainAndRefreshIndex(500);
+ app.waitForQueueDrainAndRefreshIndex(5000);
// get the receipts entity IDs
List<EntityRef> receipts = getNotificationReceipts(notification);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/3f7afcd8/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index bcc8b8e..4178a07 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -35,7 +35,7 @@ elasticsearch.managment_index=usergrid_services_management
elasticsearch.buffer_timeout=1
elasticsearch.queue_impl.resolution=true
-elasticsearch.queue_impl=DISTRIBUTED
+elasticsearch.queue_impl=LOCAL
# Queueing Test Settings
queue.long.polling.time.millis=50