You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2017/10/18 16:22:25 UTC

[43/51] [abbrv] usergrid git commit: Allow submission to SNS/SQS via sync client

Allow submission to SNS/SQS via sync client


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/9a151089
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/9a151089
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/9a151089

Branch: refs/heads/asf-site
Commit: 9a15108924b29997c9ca440b467ab8006a8f8acb
Parents: 173be8a
Author: Peter Johnson <pj...@apigee.com>
Authored: Wed Sep 27 08:19:20 2017 -0700
Committer: Peter Johnson <pj...@apigee.com>
Committed: Wed Sep 27 08:19:20 2017 -0700

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  75 +++++++------
 .../corepersistence/CpRelationManager.java      |  52 ++++-----
 .../asyncevents/AsyncEventService.java          |   6 +-
 .../asyncevents/AsyncEventServiceImpl.java      |  44 +++++---
 .../asyncevents/EventBuilderImpl.java           |   2 +
 .../corepersistence/util/CpCollectionUtils.java | 104 +++++++++++++++++
 .../index/AsyncIndexServiceTest.java            |   2 +-
 .../persistence/queue/LegacyQueueFig.java       |   5 +
 .../persistence/queue/LegacyQueueManager.java   |  11 +-
 .../persistence/queue/LocalQueueManager.java    |  11 +-
 .../queue/impl/QakkaQueueManager.java           |  12 +-
 .../queue/impl/SNSQueueManagerImpl.java         | 112 +++++++++++++++++--
 .../queue/LegacyQueueManagerTest.java           |   2 +-
 .../impl/ApplicationQueueManagerImpl.java       |   2 +-
 .../services/queues/ImportQueueManager.java     |   8 +-
 15 files changed, 341 insertions(+), 107 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 68c4ef0..1dc4a89 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -34,6 +34,7 @@ import org.apache.usergrid.corepersistence.index.CollectionSettingsFactory;
 import org.apache.usergrid.corepersistence.index.CollectionSettingsScopeImpl;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.util.CpCollectionUtils;
 import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.mq.QueueManager;
@@ -161,7 +162,6 @@ public class CpEntityManager implements EntityManager {
 
     public QueueManagerFactory queueManagerFactory;
 
-
     //    /** Short-term cache to keep us from reloading same Entity during single request. */
 //    private LoadingCache<EntityScope, org.apache.usergrid.persistence.model.entity.Entity> entityCache;
 
@@ -521,6 +521,10 @@ public class CpEntityManager implements EntityManager {
 
         cpEntity = CpEntityMapUtils.fromMap( cpEntity, entity.getProperties(), entity.getType(), true );
 
+        String entityType = cpEntity.getId().getType();
+        boolean skipIndexingForType = skipIndexingForType(entityType);
+        Boolean asyncIndex = asyncIndexingForType(entityType);
+
         try {
 
             String region = lookupAuthoritativeRegionForType( entity.getType() );
@@ -546,38 +550,32 @@ public class CpEntityManager implements EntityManager {
             handleWriteUniqueVerifyException( entity, wuve );
         }
 
-        if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
+        if (!skipIndexingForType) {
+            indexEntity(cpEntity, asyncIndex);
+            deIndexOldVersionsOfEntity(cpEntity);
+        }
+    }
 
-            // queue an event to update the new entity
-            indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+    private void indexEntity(org.apache.usergrid.persistence.model.entity.Entity cpEntity, Boolean async) {
+        // queue an event to update the new entity
+        indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 , async);
+    }
 
-            // queue up an event to clean-up older versions than this one from the index
-            if (entityManagerFig.getDeindexOnUpdate()) {
-                indexService.queueDeIndexOldVersion( applicationScope, cpEntity.getId(), cpEntity.getVersion());
-            }
+    private void deIndexOldVersionsOfEntity(org.apache.usergrid.persistence.model.entity.Entity cpEntity) {
+        // queue up an event to clean-up older versions than this one from the index
+        if (entityManagerFig.getDeindexOnUpdate()) {
+            indexService.queueDeIndexOldVersion( applicationScope, cpEntity.getId(), cpEntity.getVersion());
         }
     }
 
-    private boolean skipIndexingForType( String type ) {
-
-        boolean skipIndexing = false;
-        String collectionName = Schema.defaultCollectionName( type );
 
+    private Boolean asyncIndexingForType( String type ) {
+        return CpCollectionUtils.asyncIndexingForType(collectionSettingsFactory, applicationId, type);
 
-        CollectionSettings collectionSettings = collectionSettingsFactory
-            .getInstance( new CollectionSettingsScopeImpl(getAppIdObject(), collectionName) );
-        Optional<Map<String, Object>> existingSettings =
-            collectionSettings.getCollectionSettings( collectionName );
-
-        if ( existingSettings.isPresent()) {
-            Map jsonMapData = existingSettings.get();
-            Object fields = jsonMapData.get("fields");
-            if ( fields != null && "none".equalsIgnoreCase( fields.toString() ) ) {
-                skipIndexing = true;
-            }
-        }
+    }
 
-        return skipIndexing;
+    private boolean skipIndexingForType( String type ) {
+        return CpCollectionUtils.skipIndexingForType(collectionSettingsFactory, applicationId, type);
     }
 
 
@@ -1153,7 +1151,7 @@ public class CpEntityManager implements EntityManager {
         //Adding graphite metrics
 
         if ( !skipIndexingForType( cpEntity.getId().getType() ) ) {
-            indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 );
+            indexService.queueEntityIndexUpdate( applicationScope, cpEntity, 0 , null);
         }
     }
 
@@ -1813,9 +1811,10 @@ public class CpEntityManager implements EntityManager {
             updatedSettings.put( "lastReindexed", 0 );
         }
 
-        // if fields specified, then put in settings
-        if ( newSettings.get("fields") != null ) {
-            updatedSettings.put("fields", newSettings.get("fields"));
+        for (String validName : CpCollectionUtils.getValidSettings()) {
+            if (newSettings.containsKey(validName)) {
+                updatedSettings.put(validName, newSettings.get(validName));
+            }
         }
 
         // if region specified
@@ -2854,14 +2853,8 @@ public class CpEntityManager implements EntityManager {
         entity.setProperties( cpEntity );
 
         // add to and index in collection of the application
-        if ( !is_application ) {
-
-            String collectionName = Schema.defaultCollectionName( eType );
-            CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
-            cpr.addToCollection( collectionName, entity );
-
-            // Invoke counters
-            incrementEntityCollection( collectionName, timestamp );
+        if ( !is_application) {
+            updateIndexForEniity(eType, entity, timestamp);
         }
 
         //write to our types map
@@ -2871,6 +2864,14 @@ public class CpEntityManager implements EntityManager {
         return entity;
     }
 
+    private <A extends Entity> void updateIndexForEniity(String eType, A entity,  long timestamp) throws Exception {
+        String collectionName = Schema.defaultCollectionName( eType );
+        CpRelationManager cpr = ( CpRelationManager ) getRelationManager( getApplication() );
+        cpr.addToCollection( collectionName, entity );
+
+        // Invoke counters
+        incrementEntityCollection( collectionName, timestamp );
+    }
 
     private void incrementEntityCollection( String collection_name, long cassandraTimestamp ) {
         try {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c02ca7d..06f06ad 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -31,6 +31,7 @@ import org.apache.usergrid.corepersistence.service.CollectionSearch;
 import org.apache.usergrid.corepersistence.service.CollectionService;
 import org.apache.usergrid.corepersistence.service.ConnectionSearch;
 import org.apache.usergrid.corepersistence.service.ConnectionService;
+import org.apache.usergrid.corepersistence.util.CpCollectionUtils;
 import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.*;
@@ -349,6 +350,7 @@ public class CpRelationManager implements RelationManager {
         Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
         org.apache.usergrid.persistence.model.entity.Entity memberEntity = ( ( CpEntityManager ) em ).load( entityId );
 
+        Id memberEntityId = memberEntity.getId();
 
         // don't fetch entity if we've already got one
         final Entity itemEntity;
@@ -364,7 +366,7 @@ public class CpRelationManager implements RelationManager {
         }
 
 
-        if ( memberEntity == null ) {
+        if ( memberEntityId == null ) {
             throw new RuntimeException(
                 "Unable to load entity uuid=" + itemRef.getUuid() + " type=" + itemRef.getType() );
         }
@@ -376,7 +378,7 @@ public class CpRelationManager implements RelationManager {
 
 
         // create graph edge connection from head entity to member entity
-        final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntity.getId() );
+        final Edge edge = createCollectionEdge( cpHeadEntity.getId(), collectionName, memberEntityId );
         final String linkedCollection = collection.getLinkedCollection();
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
@@ -387,21 +389,24 @@ public class CpRelationManager implements RelationManager {
             }
         } ).filter( writtenEdge -> linkedCollection != null ).flatMap( writtenEdge -> {
             final String pluralType = InflectionUtils.pluralize( cpHeadEntity.getId().getType() );
-            final Edge reverseEdge = createCollectionEdge( memberEntity.getId(), pluralType, cpHeadEntity.getId() );
+            final Edge reverseEdge = createCollectionEdge( memberEntityId, pluralType, cpHeadEntity.getId() );
 
             //reverse
             return gm.writeEdge( reverseEdge ).doOnNext( reverseEdgeWritten -> {
 
-                if ( !skipIndexingForType( cpHeadEntity.getId().getType() ) ) {
-
-                    indexService.queueNewEdge(applicationScope, cpHeadEntity, reverseEdge);
+                String entityType = cpHeadEntity.getId().getType();
+                if ( !skipIndexingForType( entityType) ) {
+                    Boolean async = asyncIndexingForType(entityType);
+                    indexService.queueNewEdge(applicationScope, cpHeadEntity.getId(), reverseEdge, async);
                 }
 
             } );
         } ).doOnCompleted( () -> {
 
-            if ( !skipIndexingForType( memberEntity.getId().getType() ) ) {
-                indexService.queueNewEdge(applicationScope, memberEntity, edge);
+            String entityType = memberEntity.getId().getType();
+            if ( !skipIndexingForType( entityType ) ) {
+                Boolean async = asyncIndexingForType(entityType);
+                indexService.queueNewEdge(applicationScope, memberEntityId, edge, async);
             }
 
 
@@ -731,9 +736,10 @@ public class CpRelationManager implements RelationManager {
         gm.writeEdge(edge).toBlocking().lastOrDefault(null); //throw an exception if this fails
 
 
-        if ( !skipIndexingForType( targetEntity.getId().getType() ) ) {
-
-            indexService.queueNewEdge(applicationScope, targetEntity, edge);
+        String entityType = targetEntity.getId().getType();
+        if ( !skipIndexingForType( entityType ) ) {
+            Boolean async = asyncIndexingForType(entityType);
+            indexService.queueNewEdge(applicationScope, targetEntity.getId(), edge, async);
         }
 
         // remove any duplicate edges (keeps the duplicate edge with same timestamp)
@@ -1094,27 +1100,13 @@ public class CpRelationManager implements RelationManager {
 
     }
 
-    private boolean skipIndexingForType( String type ) {
-
-        boolean skipIndexing = false;
-
-        String collectionName = Schema.defaultCollectionName( type );
+    private Boolean asyncIndexingForType( String type ) {
+        return CpCollectionUtils.asyncIndexingForType(collectionSettingsFactory, applicationId, type);
 
-        CollectionSettings collectionSettings =
-            collectionSettingsFactory.
-                getInstance( new CollectionSettingsScopeImpl(new SimpleId( applicationId, TYPE_APPLICATION ), collectionName ) );
-        Optional<Map<String, Object>> collectionIndexingSchema =
-            collectionSettings.getCollectionSettings( collectionName );
-
-        if ( collectionIndexingSchema.isPresent()) {
-            Map jsonMapData = collectionIndexingSchema.get();
-            final Object fields = jsonMapData.get( "fields" );
-            if ( fields != null && fields instanceof String && "none".equalsIgnoreCase( fields.toString())) {
-                skipIndexing = true;
-            }
-        }
+    }
 
-        return skipIndexing;
+    private boolean skipIndexingForType( String type ) {
+        return CpCollectionUtils.skipIndexingForType(collectionSettingsFactory, applicationId, type);
     }
 
     /**

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
index 04eaf4c..1ddbac4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventService.java
@@ -53,7 +53,7 @@ public interface AsyncEventService extends ReIndexAction, CollectionDeleteAction
      * @param entity The entity to index.  Should be fired when an entity is updated
      * @param updatedAfter
      */
-    void queueEntityIndexUpdate(final ApplicationScope applicationScope, final Entity entity, long updatedAfter);
+    void queueEntityIndexUpdate(final ApplicationScope applicationScope, final Entity entity, long updatedAfter, Boolean async);
 
 
     /**
@@ -63,10 +63,10 @@ public interface AsyncEventService extends ReIndexAction, CollectionDeleteAction
      * TODO: We shouldn't take an entity here, only the id. It doesn't make sense in a distributed context
      *
      * @param applicationScope
-     * @param entity
+     * @param entityId
      * @param newEdge
      */
-    void queueNewEdge(final ApplicationScope applicationScope, final Entity entity, final Edge newEdge);
+    void queueNewEdge(final ApplicationScope applicationScope, final Id entityId, final Edge newEdge, Boolean async);
 
     /**
      * Queue the deletion of an edge

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
index 3d06cae..3e67110 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/AsyncEventServiceImpl.java
@@ -276,15 +276,26 @@ public class AsyncEventServiceImpl implements AsyncEventService {
      * Offer the EntityIdScope to SQS
      */
     private void offer(final Serializable operation) {
-        offer(operation, AsyncEventQueueType.REGULAR);
+        offer(operation, AsyncEventQueueType.REGULAR, null);
     }
 
-    private void offer(final Serializable operation, AsyncEventQueueType queueType) {
+    /**
+     * Offer the EntityIdScope to SQS
+     */
+    private void offer(final Serializable operation, Boolean async) {
+        offer(operation, AsyncEventQueueType.REGULAR, async);
+    }
+
+     /**
+      * Offer the EntityIdScope to SQS
+      */
+    private void offer(final Serializable operation, AsyncEventQueueType queueType, Boolean async) {
         final Timer.Context timer = this.writeTimer.time();
 
         try {
             //signal to SQS
-            getQueue(queueType).sendMessageToLocalRegion(operation);
+            getQueue(queueType).sendMessageToLocalRegion(operation, async);
+
         } catch (IOException e) {
             throw new RuntimeException("Unable to queue message", e);
         } finally {
@@ -299,7 +310,8 @@ public class AsyncEventServiceImpl implements AsyncEventService {
 
         try {
             //signal to SQS
-            getQueue(queueType).sendMessageToAllRegions(operation);
+            getQueue(queueType).sendMessageToAllRegions(operation,null);
+
         }
         catch ( IOException e ) {
             throw new RuntimeException( "Unable to queue message", e );
@@ -530,7 +542,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
 
     @Override
     public void queueEntityIndexUpdate(final ApplicationScope applicationScope,
-                                       final Entity entity, long updatedAfter) {
+                                       final Entity entity, long updatedAfter, Boolean async) {
 
 
         if (logger.isTraceEnabled()) {
@@ -538,13 +550,18 @@ public class AsyncEventServiceImpl implements AsyncEventService {
                 entity.getId().getUuid(), entity.getId().getType());
         }
 
-        offer(new EntityIndexEvent(queueFig.getPrimaryRegion(),
-            new EntityIdScope(applicationScope, entity.getId()), updatedAfter));
+
+        EntityIndexEvent event = new EntityIndexEvent(queueFig.getPrimaryRegion(),
+            new EntityIdScope(applicationScope, entity.getId()),
+            updatedAfter);
+
+        offer(event, async);
 
     }
 
     private IndexOperationMessage handleEntityIndexUpdate(final LegacyQueueMessage message) {
 
+
         Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEntityIndexUpdate" );
 
         final AsyncEvent event = ( AsyncEvent ) message.getBody();
@@ -574,15 +591,16 @@ public class AsyncEventServiceImpl implements AsyncEventService {
 
     @Override
     public void queueNewEdge(final ApplicationScope applicationScope,
-                             final Entity entity,
-                             final Edge newEdge) {
+                             final Id entityId,
+                             final Edge newEdge,
+                             Boolean async) {
 
         if (logger.isTraceEnabled()) {
             logger.trace("Offering EdgeIndexEvent for edge type {} entity {}:{}",
-                newEdge.getType(), entity.getId().getUuid(), entity.getId().getType());
+                newEdge.getType(), entityId.getUuid(), entityId.getType());
         }
 
-        offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entity.getId(), newEdge ));
+        offer( new EdgeIndexEvent( queueFig.getPrimaryRegion(), applicationScope, entityId, newEdge ), async);
 
     }
 
@@ -620,7 +638,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
         }
 
         // sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
-        offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE );
+        offer( new EdgeDeleteEvent( queueFig.getPrimaryRegion(), applicationScope, edge ), AsyncEventQueueType.DELETE , null);
     }
 
     private IndexOperationMessage  handleEdgeDelete(final LegacyQueueMessage message) {
@@ -824,7 +842,7 @@ public class AsyncEventServiceImpl implements AsyncEventService {
 
         // sent in region (not offerTopic) as the delete IO happens in-region, then queues a multi-region de-index op
         offer( new EntityDeleteEvent(queueFig.getPrimaryRegion(), new EntityIdScope( applicationScope, entityId ) ),
-            AsyncEventQueueType.DELETE );
+            AsyncEventQueueType.DELETE , null);
     }
 
     private IndexOperationMessage handleEntityDelete(final LegacyQueueMessage message) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
index 7c72b72..203d32a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/asyncevents/EventBuilderImpl.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.corepersistence.asyncevents;
 
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.UUID;
 
@@ -303,6 +304,7 @@ public class EventBuilderImpl implements EventBuilder {
         return indexService.deIndexOldVersions( applicationScope, entityId,
             getVersionsOlderThanOrEqualToMarked(ecm, entityId, markedVersion));
 
+
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java
new file mode 100644
index 0000000..cef6d12
--- /dev/null
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpCollectionUtils.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2014 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.usergrid.corepersistence.util;
+
+import com.google.common.base.Optional;
+import org.apache.usergrid.corepersistence.index.CollectionSettings;
+import org.apache.usergrid.corepersistence.index.CollectionSettingsFactory;
+import org.apache.usergrid.corepersistence.index.CollectionSettingsScopeImpl;
+
+import org.apache.usergrid.persistence.*;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+
+import java.util.*;
+
+import static org.apache.usergrid.persistence.Schema.*;
+
+
+/**
+ *
+ * Helper methods to manage the Collection setting properties
+ *
+ */
+public class CpCollectionUtils {
+
+
+    public static final String SETTING_FIELDS = "fields";
+    public static final String SETTING_QUEUE_INDEX = "queueIndex";
+
+    private static Set<String> VALID_SETTING_NAMES = new HashSet<>();
+
+    static {
+        VALID_SETTING_NAMES.add(SETTING_FIELDS);
+        VALID_SETTING_NAMES.add(SETTING_QUEUE_INDEX);
+    }
+
+    public static Set<String> getValidSettings() {
+        return VALID_SETTING_NAMES;
+    }
+
+    public static Boolean asyncIndexingForType(CollectionSettingsFactory collectionSettingsFactory, UUID applicationId, String type ) {
+
+        String indexing = getFieldForType(applicationId, collectionSettingsFactory, type, SETTING_QUEUE_INDEX);
+        if ("async".equals(indexing)) {
+            return Boolean.TRUE;
+        }
+        if ("sync".equals(indexing)) {
+            return Boolean.FALSE;
+        }
+        return null;
+    }
+
+    public static boolean skipIndexingForType(CollectionSettingsFactory collectionSettingsFactory, UUID applicationId, String type ) {
+
+        String fields = getFieldForType(applicationId, collectionSettingsFactory, type, SETTING_FIELDS);
+        boolean skipIndexing = false;
+        if ( fields != null && fields instanceof String && "none".equalsIgnoreCase( fields.toString())) {
+            skipIndexing = true;
+        }
+
+        return skipIndexing;
+    }
+
+    // these same methods are in CpEntityManager must refactor
+    private static String getFieldForType(UUID applicationId, CollectionSettingsFactory collectionSettingsFactory,
+                                          String type, String keyName ) {
+
+        String collectionName = Schema.defaultCollectionName( type );
+
+        CollectionSettings collectionSettings = collectionSettingsFactory
+            .getInstance( new CollectionSettingsScopeImpl(getAppIdObject(applicationId), collectionName) );
+        Optional<Map<String, Object>> existingSettings =
+            collectionSettings.getCollectionSettings( collectionName );
+
+        if ( existingSettings.isPresent()) {
+            Map jsonMapData = existingSettings.get();
+            Object value = jsonMapData.get(keyName);
+            if ( value != null) {
+                return value.toString();
+            }
+        }
+        return null;
+    }
+
+    private static Id getAppIdObject(UUID applicationId){
+        return new SimpleId( applicationId, TYPE_APPLICATION );
+    }
+
+
+}

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
index cecc3b2..766e2b2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/index/AsyncIndexServiceTest.java
@@ -144,7 +144,7 @@ public abstract class AsyncIndexServiceTest {
 
 
         //queue up processing
-        asyncEventService.queueEntityIndexUpdate( applicationScope, testEntity, 0);
+        asyncEventService.queueEntityIndexUpdate( applicationScope, testEntity, 0, null);
 
 
         final EntityIndex EntityIndex =

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
index 0ebcc7b..f19bede 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueFig.java
@@ -109,4 +109,9 @@ public interface LegacyQueueFig extends GuicyFig {
     @Key("usergrid.queue.map.message.timeout")
     @Default("900000") // 15 minutes
     int getMapMessageTimeout();
+
+    @Key("usergrid.queue.is.async")
+    @Default("true")
+    boolean isAsyncQueue();
+
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
index f153610..148cb5d 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LegacyQueueManager.java
@@ -66,6 +66,13 @@ public interface LegacyQueueManager {
      * @param bodies body objects must be serializable
      * @throws IOException
      */
+    void sendMessagesAsync(List bodies) throws IOException;
+
+    /**
+     * send messages to queue
+     * @param bodies body objects must be serializable
+     * @throws IOException
+     */
     void sendMessages(List bodies) throws IOException;
 
     /**
@@ -81,13 +88,13 @@ public interface LegacyQueueManager {
      * @param body
      * @throws IOException
      */
-    <T extends Serializable> void sendMessageToLocalRegion(T body)throws IOException;
+    <T extends Serializable> void sendMessageToLocalRegion(T body, Boolean async)throws IOException;
 
     /**
      * Send a messae to the topic to be sent to other queues
      * @param body
      */
-    <T extends Serializable> void sendMessageToAllRegions(T body) throws IOException;
+    <T extends Serializable> void sendMessageToAllRegions(T body, Boolean async) throws IOException;
 
     /**
      * purge messages

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
index cbba0b1..1f07b58 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/LocalQueueManager.java
@@ -79,6 +79,11 @@ public class LocalQueueManager implements LegacyQueueManager {
     }
 
     @Override
+    public void sendMessagesAsync(List bodies) throws IOException {
+        sendMessages(bodies);
+    }
+
+    @Override
     public  void sendMessages(List bodies) throws IOException {
         for(Object body : bodies){
             String uuid = UUID.randomUUID().toString();
@@ -108,7 +113,7 @@ public class LocalQueueManager implements LegacyQueueManager {
 
 
     @Override
-    public <T extends Serializable> void sendMessageToLocalRegion(final T body ) throws IOException {
+    public <T extends Serializable> void sendMessageToLocalRegion(final T body, Boolean async) throws IOException {
         String uuid = UUID.randomUUID().toString();
         try {
             queue.offer(new LegacyQueueMessage(uuid, "handle_" + uuid, body, "put type here"),5000,TimeUnit.MILLISECONDS);
@@ -120,8 +125,8 @@ public class LocalQueueManager implements LegacyQueueManager {
 
 
     @Override
-    public <T extends Serializable> void sendMessageToAllRegions(final T body ) throws IOException {
-       sendMessageToLocalRegion( body );
+    public <T extends Serializable> void sendMessageToAllRegions(final T body, Boolean async) throws IOException {
+       sendMessageToLocalRegion( body, null );
     }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
index e7fa47b..59110df 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/QakkaQueueManager.java
@@ -101,7 +101,7 @@ public class QakkaQueueManager implements LegacyQueueManager {
 
 
     @Override
-    public <T extends Serializable> void sendMessageToLocalRegion(T body) throws IOException {
+    public <T extends Serializable> void sendMessageToLocalRegion(T body, Boolean async) throws IOException {
         List<String> regionsList = regions.getRegions( Regions.LOCAL );
         logger.trace( "Sending message to queue {} local region {}", scope.getName(), regionsList );
         doSendMessage( body, regionsList );
@@ -109,7 +109,7 @@ public class QakkaQueueManager implements LegacyQueueManager {
 
 
     @Override
-    public <T extends Serializable> void sendMessageToAllRegions(T body) throws IOException {
+    public <T extends Serializable> void sendMessageToAllRegions(T body, Boolean async) throws IOException {
         List<String> regionsList = regions.getRegions( Regions.ALL );
         logger.trace( "Sending message to queue {} all regions {}", scope.getName(), regionsList );
         doSendMessage( body, regionsList );
@@ -183,12 +183,16 @@ public class QakkaQueueManager implements LegacyQueueManager {
         }
     }
 
+    @Override
+    public void sendMessagesAsync( List bodies ) throws IOException {
+        sendMessages(bodies);
+    }
 
     @Override
     public void sendMessages( List bodies ) throws IOException {
 
         for ( Object body : bodies ) {
-            sendMessageToLocalRegion( (Serializable)body );
+            sendMessageToLocalRegion( (Serializable)body, null );
         }
 
     }
@@ -199,7 +203,7 @@ public class QakkaQueueManager implements LegacyQueueManager {
 
         List<LegacyQueueMessage> successMessages = new ArrayList<>();
         for ( LegacyQueueMessage queueMessage : queueMessages ) {
-            sendMessageToLocalRegion( (Serializable)queueMessage.getBody() );
+            sendMessageToLocalRegion( (Serializable)queueMessage.getBody() , null);
             successMessages.add(queueMessage);
         }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
index 5b49bc7..775a64b 100644
--- a/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
+++ b/stack/corepersistence/queue/src/main/java/org/apache/usergrid/persistence/queue/impl/SNSQueueManagerImpl.java
@@ -541,7 +541,49 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
 
 
     @Override
-    public <T extends Serializable> void sendMessageToAllRegions(final T body ) throws IOException {
+    public <T extends Serializable> void sendMessageToAllRegions(final T body, Boolean async) throws IOException {
+        boolean sendAsync = async == null ? fig.isAsyncQueue() : async.booleanValue();
+        if (sendAsync) {
+            sendMessageToAllRegionsAsync(body);
+        } else {
+            sendMessageToAllRegionsSync(body);
+        }
+    }
+
+
+    private <T extends Serializable> void sendMessageToAllRegionsSync(final T body) throws IOException {
+        if ( sns == null ) {
+            logger.error( "SNS client is null, perhaps it failed to initialize successfully" );
+            return;
+        }
+
+        final String stringBody = toString( body );
+
+        String topicArn = getWriteTopicArn();
+
+        if ( logger.isTraceEnabled() ) {
+            logger.trace( "Publishing Message...{} to arn: {}", stringBody, topicArn );
+        }
+
+        try {
+            PublishResult publishResult = sns.publish(topicArn, toString(body));
+            if ( logger.isTraceEnabled() ) {
+                logger.trace( "Successfully published... messageID=[{}],  arn=[{}]", publishResult.getMessageId(),
+                    topicArn );
+            }
+        } catch (Exception e) {
+            if (logger.isErrorEnabled()) {
+                logger.error("Failed to send this message {} to SNS queue at {}", stringBody, topicArn);
+            }
+            sendMessageToAllRegionsAsync(body);
+        }
+
+
+
+    }
+
+
+    private <T extends Serializable> void sendMessageToAllRegionsAsync(final T body ) throws IOException {
         if ( snsAsync == null ) {
             logger.error( "SNS client is null, perhaps it failed to initialize successfully" );
             return;
@@ -574,17 +616,27 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
         } );
     }
 
-
     @Override
-    public void sendMessages( final List bodies ) throws IOException {
-
+    public void sendMessagesAsync( final List bodies ) throws IOException {
         if ( sqsAsync == null ) {
             logger.error( "SQS client is null, perhaps it failed to initialize successfully" );
             return;
         }
 
         for ( Object body : bodies ) {
-            sendMessageToLocalRegion( ( Serializable ) body );
+            sendMessageToLocalRegionAsync( ( Serializable ) body );
+        }
+    }
+
+
+    @Override
+    public void sendMessages( final List bodies ) throws IOException {
+        for ( Object body : bodies ) {
+            if (fig.isAsyncQueue()) {
+                sendMessageToLocalRegionAsync((Serializable) body);
+            } else {
+                sendMessageToLocalRegionSync((Serializable) body);
+            }
         }
     }
 
@@ -625,16 +677,57 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
         return successMessages;
     }
 
-
     @Override
-    public <T extends Serializable> void sendMessageToLocalRegion(final T body ) throws IOException {
+    public <T extends Serializable> void sendMessageToLocalRegion(final T body, Boolean async) throws IOException {
+        boolean sendAsync = async == null ? fig.isAsyncQueue() : async.booleanValue();
+        if (sendAsync) {
+            sendMessageToLocalRegionAsync(body);
+        } else {
+            sendMessageToLocalRegionSync(body);
+        }
+    }
 
-        if ( sqsAsync == null ) {
+    private <T extends Serializable> void sendMessageToLocalRegionSync(final T body) throws IOException {
+
+        if ( sqs == null ) {
             logger.error( "SQS client is null, perhaps it failed to initialize successfully" );
             return;
         }
         final String stringBody = toString( body );
 
+        if (logger.isDebugEnabled()) {
+            logger.debug(" sendMessageToLocalRegion " + stringBody);
+        }
+
+        String url = getReadQueue().getUrl();
+
+        if ( logger.isTraceEnabled() ) {
+            logger.trace( "Publishing Message...{} to url: {}", stringBody, url );
+        }
+
+        SendMessageRequest messageRequest = new SendMessageRequest(url, stringBody);
+        try {
+            SendMessageResult result = sqs.sendMessage(messageRequest);
+            if (logger.isTraceEnabled()) {
+                logger.trace("Successfully published... messageID=[{}],  arn=[{}]", result.getMessageId(),
+                    url);
+            }
+        } catch (Exception e) {
+            logger.error("Failed to send this message {}. To this address {}. Error was ",  messageRequest.getMessageBody(), url, e);
+            sendMessageToLocalRegionAsync(body);
+        }
+
+
+    }
+
+
+    private <T extends Serializable> void sendMessageToLocalRegionAsync(final T body ) throws IOException {
+
+        if ( sqsAsync == null ) {
+            logger.error( "SQS client is null, perhaps it failed to initialize successfully" );
+            return;
+        }
+        final String stringBody = toString( body );
         String url = getReadQueue().getUrl();
 
         if ( logger.isTraceEnabled() ) {
@@ -647,8 +740,7 @@ public class SNSQueueManagerImpl implements LegacyQueueManager {
 
             @Override
             public void onError( final Exception e ) {
-
-                logger.error( "Error sending message... {}", e );
+                logger.error("Failed to send this message {}. To this address {}. Error was ", stringBody, url, e);
             }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
index ad73dd7..c57ff8b 100644
--- a/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
+++ b/stack/corepersistence/queue/src/test/java/org/apache/usergrid/persistence/queue/LegacyQueueManagerTest.java
@@ -62,7 +62,7 @@ public class LegacyQueueManagerTest extends AbstractAkkaTest {
         LegacyQueueManager qm = qmf.getQueueManager(scope);
 
         String value = "bodytest";
-        qm.sendMessageToLocalRegion(value);
+        qm.sendMessageToLocalRegion(value, null);
 
         Thread.sleep(5000);
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
index d26cd5f..b7afc2e 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/notifications/impl/ApplicationQueueManagerImpl.java
@@ -356,7 +356,7 @@ public class ApplicationQueueManagerImpl implements ApplicationQueueManager {
                             if(logger.isTraceEnabled()) {
                                 logger.trace("Queueing notification message for device: {}", message.get().getDeviceId());
                             }
-                            qm.sendMessageToLocalRegion( message.get() );
+                            qm.sendMessageToLocalRegion( message.get() , null );
                             queueMeter.mark();
                         }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/9a151089/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
----------------------------------------------------------------------
diff --git a/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java b/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
index f7e107d..ae58d7d 100644
--- a/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
+++ b/stack/services/src/main/java/org/apache/usergrid/services/queues/ImportQueueManager.java
@@ -59,6 +59,10 @@ public class ImportQueueManager implements LegacyQueueManager {
 
     }
 
+    @Override
+    public void sendMessagesAsync( final List bodies ) throws IOException {
+
+    }
 
     @Override
     public void sendMessages( final List bodies ) throws IOException {
@@ -73,13 +77,13 @@ public class ImportQueueManager implements LegacyQueueManager {
 
 
     @Override
-    public <T extends Serializable> void sendMessageToLocalRegion(final T body ) throws IOException {
+    public <T extends Serializable> void sendMessageToLocalRegion(final T body, Boolean async) throws IOException {
 
     }
 
 
     @Override
-    public <T extends Serializable> void sendMessageToAllRegions(final T body ) throws IOException {
+    public <T extends Serializable> void sendMessageToAllRegions(final T body, Boolean async) throws IOException {
 
     }