You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/09/21 00:12:42 UTC

[08/21] usergrid git commit: remove batch.execute

remove batch.execute


Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c03a7298
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c03a7298
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c03a7298

Branch: refs/heads/two-dot-o-dev
Commit: c03a729817dd7bf5e336a6fe4c9be5984e4b3fb5
Parents: c05d973
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Sep 16 08:21:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Sep 16 08:21:59 2015 -0600

----------------------------------------------------------------------
 .../corepersistence/CpEntityManager.java        |  6 +-
 .../corepersistence/CpEntityManagerFactory.java |  5 +-
 .../corepersistence/CpRelationManager.java      | 16 ++---
 .../corepersistence/index/IndexServiceImpl.java | 22 ++++---
 .../read/search/CandidateEntityFilter.java      | 14 +++--
 .../pipeline/read/search/CandidateIdFilter.java | 14 +++--
 .../persistence/index/EntityIndexBatch.java     |  7 ++-
 .../index/impl/EsEntityIndexBatchImpl.java      | 14 ++---
 .../index/impl/EsIndexProducerImpl.java         |  9 ++-
 .../persistence/index/impl/IndexProducer.java   |  2 +
 .../persistence/index/impl/EntityIndexTest.java | 66 ++++++++++----------
 .../persistence/index/impl/GeoPagingTest.java   |  5 +-
 .../index/impl/IndexLoadTestsIT.java            |  6 +-
 13 files changed, 115 insertions(+), 71 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index d46c112..e222f68 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -41,6 +41,7 @@ import org.apache.usergrid.corepersistence.service.ConnectionService;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.IndexLocationStrategy;
 import org.apache.usergrid.persistence.index.IndexRefreshCommand;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -183,6 +184,7 @@ public class CpEntityManager implements EntityManager {
 
     private final UUID applicationId;
     private final EntityManagerFig entityManagerFig;
+    private final IndexProducer indexProducer;
     private Application application;
 
 
@@ -246,9 +248,11 @@ public class CpEntityManager implements EntityManager {
                             final GraphManagerFactory graphManagerFactory,
                             final CollectionService collectionService,
                             final ConnectionService connectionService,
+                            final IndexProducer indexProducer,
                             final UUID applicationId ) {
 
         this.entityManagerFig = entityManagerFig;
+        this.indexProducer = indexProducer;
 
         Preconditions.checkNotNull( cass, "cass must not be null" );
         Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
@@ -759,7 +763,7 @@ public class CpEntityManager implements EntityManager {
         Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
 
         CpRelationManager relationManager =
-            new CpRelationManager( managerCache, indexService, collectionService, connectionService,  this, entityManagerFig, applicationId, entityRef );
+            new CpRelationManager(managerCache, indexService, collectionService, connectionService, indexProducer, this, entityManagerFig, applicationId, entityRef );
         return relationManager;
     }
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 4bdade5..4a712a7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -22,6 +22,7 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.BeansException;
@@ -96,6 +97,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
 
     private static final Logger logger = LoggerFactory.getLogger( CpEntityManagerFactory.class );
     private final EntityManagerFig entityManagerFig;
+    private final IndexProducer indexProducer;
 
     private ApplicationContext applicationContext;
 
@@ -137,6 +139,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
         this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class );
         this.collectionService = injector.getInstance( CollectionService.class );
         this.connectionService = injector.getInstance( ConnectionService.class );
+        this.indexProducer = injector.getInstance(IndexProducer.class);
 
         //this line always needs to be last due to the temporary cicular dependency until spring is removed
         this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
@@ -199,7 +202,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
 
     private EntityManager _getEntityManager( UUID applicationId ) {
         EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
-            metricsFactory, entityManagerFig, graphManagerFactory,  collectionService, connectionService, applicationId );
+            metricsFactory, entityManagerFig, graphManagerFactory,  collectionService, connectionService, indexProducer, applicationId );
 
         return em;
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 0c3b927..c393fb2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -26,6 +26,7 @@ import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
 
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.util.Assert;
@@ -105,6 +106,7 @@ public class CpRelationManager implements RelationManager {
 
     private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
     private final EntityManagerFig entityManagerFig;
+    private final IndexProducer indexProducer;
 
     private ManagerCache managerCache;
 
@@ -127,9 +129,11 @@ public class CpRelationManager implements RelationManager {
 
     public CpRelationManager( final ManagerCache managerCache,
                               final AsyncEventService indexService, final CollectionService collectionService,
-                              final ConnectionService connectionService, final EntityManager em,
+                              final ConnectionService connectionService,final IndexProducer indexProducer,
+                              final EntityManager em,
                               final EntityManagerFig entityManagerFig, final UUID applicationId,
-                              final EntityRef headEntity ) {
+                              final EntityRef headEntity) {
+        this.indexProducer = indexProducer;
 
 
         Assert.notNull( em, "Entity manager cannot be null" );
@@ -529,16 +533,14 @@ public class CpRelationManager implements RelationManager {
          */
 
         final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
-        final EntityIndexBatch batch = ei.createBatch();
 
         // remove item from collection index
         SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collectionName );
 
+        final EntityIndexBatch batch = ei.createBatch();
         batch.deindex( indexScope, memberEntity );
-
-
-        batch.execute();
-
+        //TODO: this should not happen here, needs to go to  SQS
+        indexProducer.put(batch).subscribe();
 
         // special handling for roles collection of a group
         if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 8a8dba1..be7dc20 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
 import java.util.UUID;
 
 import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.apache.usergrid.utils.UUIDUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -68,18 +69,25 @@ public class IndexServiceImpl implements IndexService {
     private final EdgesObservable edgesObservable;
     private final IndexFig indexFig;
     private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+    private final IndexProducer indexProducer;
     private final Timer indexTimer;
     private final Timer addTimer;
 
 
     @Inject
-    public IndexServiceImpl( final GraphManagerFactory graphManagerFactory, final EntityIndexFactory entityIndexFactory,
-                             final EdgesObservable edgesObservable, final IndexFig indexFig, final IndexLocationStrategyFactory indexLocationStrategyFactory, final MetricsFactory metricsFactory ) {
+    public IndexServiceImpl( final GraphManagerFactory graphManagerFactory,
+                             final EntityIndexFactory entityIndexFactory,
+                             final EdgesObservable edgesObservable,
+                             final IndexFig indexFig,
+                             final IndexLocationStrategyFactory indexLocationStrategyFactory,
+                             final MetricsFactory metricsFactory,
+                             final IndexProducer indexProducer) {
         this.graphManagerFactory = graphManagerFactory;
         this.entityIndexFactory = entityIndexFactory;
         this.edgesObservable = edgesObservable;
         this.indexFig = indexFig;
         this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+        this.indexProducer = indexProducer;
         this.indexTimer = metricsFactory.getTimer( IndexServiceImpl.class, "index.update_all");
         this.addTimer = metricsFactory.getTimer( IndexServiceImpl.class, "index.add" );
     }
@@ -89,7 +97,7 @@ public class IndexServiceImpl implements IndexService {
     public Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope,
                                                           final Entity entity ) {
         //bootstrap the lower modules from their caches
-        final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+        final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
         final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope));
 
 
@@ -114,7 +122,7 @@ public class IndexServiceImpl implements IndexService {
                     batch.index( indexEdge, entity );
                 } )
                     //return the future from the batch execution
-                .flatMap( batch -> batch.execute() ) );
+                .flatMap( batch -> indexProducer.put(batch) ) );
 
         return ObservableTimer.time( batches, indexTimer );
     }
@@ -142,7 +150,7 @@ public class IndexServiceImpl implements IndexService {
 
             batch.index( indexEdge, entity );
 
-            return batch.execute();
+            return indexProducer.put(batch);
         } );
 
         return ObservableTimer.time( batches, addTimer  );
@@ -185,7 +193,7 @@ public class IndexServiceImpl implements IndexService {
 
                 batch = deindexBatchIteratorResolver( fromTarget, sourceEdgesToBeDeindexed, batch );
 
-                return batch.execute();
+                return indexProducer.put(batch);
             } );
 
         return ObservableTimer.time( batches, addTimer );
@@ -221,7 +229,7 @@ public class IndexServiceImpl implements IndexService {
                     batch.deindex( searchEdge, candidateResult );
                 } )
                     //return the future from the batch execution
-                .flatMap( batch -> batch.execute() );
+                .flatMap( batch -> indexProducer.put(batch) );
 
         return ObservableTimer.time(batches, indexTimer);
     }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 14c880f..ac2962a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -24,6 +24,7 @@ import java.util.*;
 
 import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
 import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.apache.usergrid.persistence.model.field.Field;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -56,16 +57,19 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
     private final EntityCollectionManagerFactory entityCollectionManagerFactory;
     private final EntityIndexFactory entityIndexFactory;
     private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+    private final IndexProducer producer;
 
 
     @Inject
     public CandidateEntityFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
                                   final EntityIndexFactory entityIndexFactory,
-                                  final IndexLocationStrategyFactory indexLocationStrategyFactory
+                                  final IndexLocationStrategyFactory indexLocationStrategyFactory,
+                                  final IndexProducer producer
                                   ) {
         this.entityCollectionManagerFactory = entityCollectionManagerFactory;
         this.entityIndexFactory = entityIndexFactory;
         this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+        this.producer = producer;
     }
 
 
@@ -107,7 +111,7 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
                             .flatMap(idList -> entityCollectionManager.load(idList));
                         //now we have a collection, validate our canidate set is correct.
                         return entitySets.map(
-                            entitySet -> new EntityVerifier(
+                            entitySet -> new EntityVerifier(producer,
                                 applicationIndex.createBatch(), entitySet, candidateResults)
                         )
                             .doOnNext(entityCollector -> entityCollector.merge())
@@ -148,13 +152,15 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
         private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
         private List<FilterResult<Entity>> results = new ArrayList<>();
 
+        private final IndexProducer producer;
         private final EntityIndexBatch batch;
         private final List<FilterResult<Candidate>> candidateResults;
         private final EntitySet entitySet;
 
 
-        public EntityVerifier( final EntityIndexBatch batch, final EntitySet entitySet,
+        public EntityVerifier( final IndexProducer producer, final EntityIndexBatch batch, final EntitySet entitySet,
                                final List<FilterResult<Candidate>> candidateResults ) {
+            this.producer = producer;
             this.batch = batch;
             this.entitySet = entitySet;
             this.candidateResults = candidateResults;
@@ -171,7 +177,7 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
                 validate( candidateResult );
             }
 
-           batch.execute().subscribe();
+            producer.put(batch).subscribe();
         }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
index 3b1c102..b00efee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
@@ -26,6 +26,7 @@ import java.util.UUID;
 
 import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
 import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -53,15 +54,18 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
     private final EntityCollectionManagerFactory entityCollectionManagerFactory;
     private final EntityIndexFactory entityIndexFactory;
     private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+    private final IndexProducer indexProducer;
 
 
     @Inject
     public CandidateIdFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
                               final EntityIndexFactory entityIndexFactory,
-                              final IndexLocationStrategyFactory indexLocationStrategyFactory) {
+                              final IndexLocationStrategyFactory indexLocationStrategyFactory,
+                              final IndexProducer indexProducer) {
         this.entityCollectionManagerFactory = entityCollectionManagerFactory;
         this.entityIndexFactory = entityIndexFactory;
         this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+        this.indexProducer = indexProducer;
     }
 
 
@@ -96,7 +100,7 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
                     //now we have a collection, validate our canidate set is correct.
 
                     return versionSetObservable.map(
-                        entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet,
+                        entitySet -> new EntityCollector(indexProducer, applicationIndex.createBatch(), entitySet,
                             candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
                         entityCollector -> Observable.from( entityCollector.collectResults() ) );
                 } );
@@ -113,13 +117,15 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
         private static final Logger logger = LoggerFactory.getLogger( EntityCollector.class );
         private List<FilterResult<Id>> results = new ArrayList<>();
 
+        private final IndexProducer indexProducer;
         private final EntityIndexBatch batch;
         private final List<FilterResult<Candidate>> candidateResults;
         private final VersionSet versionSet;
 
 
-        public EntityCollector( final EntityIndexBatch batch, final VersionSet versionSet,
+        public EntityCollector( final IndexProducer indexProducer, final EntityIndexBatch batch, final VersionSet versionSet,
                                 final List<FilterResult<Candidate>> candidateResults ) {
+            this.indexProducer = indexProducer;
             this.batch = batch;
             this.versionSet = versionSet;
             this.candidateResults = candidateResults;
@@ -136,7 +142,7 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
                 validate( candidateResult );
             }
 
-            batch.execute();
+            indexProducer.put(batch).subscribe();
         }
 
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index d1b076d..baca504 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -63,10 +63,11 @@ public interface EntityIndexBatch {
 
 
     /**
-     * Execute the batch
-     * @return future to guarantee execution
+     * return built payload
+     * @return
      */
-    Observable<IndexOperationMessage> execute();
+    IndexOperationMessage build();
+
 
     /**
      * Get the number of operations in the batch

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 64a1c6a..3679887 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -66,8 +66,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
     @Override
     public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity ) {
-        IndexValidationUtils.validateIndexEdge( indexEdge );
-        ValidationUtils.verifyEntityWrite( entity );
+        IndexValidationUtils.validateIndexEdge(indexEdge);
+        ValidationUtils.verifyEntityWrite(entity);
         ValidationUtils.verifyVersion( entity.getVersion() );
 
         final String writeAlias = alias.getWriteAlias();
@@ -78,7 +78,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         }
 
         //add app id for indexing
-        container.addIndexRequest( new IndexOperation( writeAlias, applicationScope, indexEdge, entity ) );
+        container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity));
         return this;
     }
 
@@ -115,18 +115,16 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
 
     @Override
-    public EntityIndexBatch deindex( final SearchEdge searchEdge, final CandidateResult entity ) {
+    public EntityIndexBatch deindex(final SearchEdge searchEdge, final CandidateResult entity) {
 
         return deindex( searchEdge, entity.getId(), entity.getVersion() );
     }
 
-
     @Override
-    public Observable execute() {
-        return indexBatchBufferProducer.put( container );
+    public IndexOperationMessage build(){
+        return container;
     }
 
-
     @Override
     public int size() {
         return container.getDeIndexRequests().size() + container.getIndexRequests().size();

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index efce823..ef59abb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -24,6 +24,7 @@ import java.util.Set;
 import java.util.concurrent.atomic.AtomicLong;
 
 import com.codahale.metrics.Histogram;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.elasticsearch.action.WriteConsistencyLevel;
 import org.elasticsearch.action.bulk.BulkItemResponse;
 import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -87,8 +88,14 @@ public class EsIndexProducerImpl implements IndexProducer {
 
     }
 
+    public Observable<IndexOperationMessage>  put( EntityIndexBatch batch ) {
+        Preconditions.checkNotNull(batch, "Batch cannot be null");
+        return put(batch.build());
+    }
+
     public Observable<IndexOperationMessage>  put( IndexOperationMessage message ) {
-        Preconditions.checkNotNull(message, "Message cannot be null");
+
+            Preconditions.checkNotNull(message, "Message cannot be null");
         indexSizeCounter.inc(message.getDeIndexRequests().size());
         indexSizeCounter.inc(message.getIndexRequests().size());
         return  processBatch(message);

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
index e18ac50..8f3ab79 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
@@ -20,6 +20,7 @@
 package org.apache.usergrid.persistence.index.impl;
 
 
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import rx.Observable;
 
 
@@ -34,6 +35,7 @@ public interface IndexProducer {
      * @param message
      * @return
      */
+    Observable<IndexOperationMessage>  put(EntityIndexBatch message);
     Observable<IndexOperationMessage>  put(IndexOperationMessage message);
 
 }

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 6348a44..2fd9d3b 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -77,6 +77,8 @@ public class EntityIndexTest extends BaseIT {
     @Inject
     public EntityIndexFactory eif;
 
+    @Inject
+    public IndexProducer indexProducer;
 
     @Inject
     public IndexFig fig;
@@ -133,25 +135,25 @@ public class EntityIndexTest extends BaseIT {
         entity1.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID()));
         entity1.setField( new StringField( "testfield", "test" ) );
         entity1.setField(new IntegerField("ordinal", 0));
-        entity1.setField(new UUIDField("testuuid",uuid));
+        entity1.setField(new UUIDField("testuuid", uuid));
 
 
-        batch.index( indexEdge, entity1 );
-        batch.execute().toBlocking().last();
+        batch.index(indexEdge, entity1);
+        indexProducer.put(batch).subscribe();
 
 
         Entity entity2 = new Entity( entityType );
-        EntityUtils.setVersion( entity2, UUIDGenerator.newTimeUUID() );
+        EntityUtils.setVersion(entity2, UUIDGenerator.newTimeUUID());
 
 
         List<String> list = new ArrayList<>();
-        list.add( "test" );
-        entity2.setField( new ArrayField<>( "testfield", list ) );
-        entity2.setField( new IntegerField( "ordinal", 1 ) );
+        list.add("test");
+        entity2.setField(new ArrayField<>("testfield", list));
+        entity2.setField(new IntegerField("ordinal", 1));
 
 
-        batch.index( indexEdge, entity2 );
-        batch.execute().toBlocking().last();
+        batch.index(indexEdge, entity2);
+        indexProducer.put(batch).subscribe();
 
         entityIndex.refreshAsync().toBlocking().first();
 
@@ -163,14 +165,14 @@ public class EntityIndexTest extends BaseIT {
 
         timer.stop();
 
-        assertEquals( 2, candidateResults.size() );
-        log.debug( "Query time {}ms", timer.getTime() );
+        assertEquals(2, candidateResults.size());
+        log.debug("Query time {}ms", timer.getTime());
 
         final CandidateResult candidate1 = candidateResults.get(0);
 
         //check the id and version
         assertEquals( entity1.getId(), candidate1.getId() );
-        assertEquals( entity1.getVersion(), candidate1.getVersion() );
+        assertEquals(entity1.getVersion(), candidate1.getVersion());
 
 
         final CandidateResult candidate2 = candidateResults.get(1);
@@ -215,7 +217,7 @@ public class EntityIndexTest extends BaseIT {
 
                     EntityIndexBatch batch = entityIndex.createBatch();
                     insertJsonBlob( sampleJson, batch, entityType, indexEdge, size, 0 );
-                    batch.execute().toBlocking().last();
+                    indexProducer.put(batch).subscribe();
                 }
                 catch ( Exception e ) {
                     synchronized ( failTime ) {
@@ -290,7 +292,7 @@ public class EntityIndexTest extends BaseIT {
 
         EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
         entityIndexBatch.deindex(searchEdge, crs.get(0));
-        entityIndexBatch.execute().toBlocking().last();
+        indexProducer.put(entityIndex.createBatch() ).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         //Hilda Youn
@@ -306,7 +308,7 @@ public class EntityIndexTest extends BaseIT {
         });
         EntityIndexBatch batch = entityIndex.createBatch();
         insertJsonBlob(sampleJson, batch, entityType, indexEdge, max, startIndex);
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         IndexRefreshCommandImpl.IndexRefreshCommandInfo info =  entityIndex.refreshAsync().toBlocking().first();
         long time = info.getExecutionTime();
         log.info("refresh took ms:" + time);
@@ -360,11 +362,11 @@ public class EntityIndexTest extends BaseIT {
 
 
         Entity entity = EntityIndexMapUtils.fromMap( entityMap );
-        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
-        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-        entity.setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
+        EntityUtils.setId(entity, new SimpleId( "fastcar" ) );
+        EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() );
+        entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
 
-        entityIndex.createBatch().index( searchEdge, entity ).execute().toBlocking().last();
+        indexProducer.put(entityIndex.createBatch().index( searchEdge, entity )).toBlocking().last();
         entityIndex.refreshAsync().toBlocking().first();
 
         CandidateResults candidateResults = entityIndex
@@ -373,7 +375,7 @@ public class EntityIndexTest extends BaseIT {
 
         EntityIndexBatch batch = entityIndex.createBatch();
         batch.deindex( searchEdge, entity );
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         candidateResults = entityIndex
@@ -410,7 +412,7 @@ public class EntityIndexTest extends BaseIT {
             entity[i].setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) );
 
             //index the new entity. This is where the loop will be set to create like 100 entities.
-            entityIndex.createBatch().index( searchEdge, entity[i] ).execute().toBlocking().last();
+            indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] )).toBlocking().last();
         }
         entityIndex.refreshAsync().toBlocking().first();
 
@@ -541,7 +543,7 @@ public class EntityIndexTest extends BaseIT {
         EntityIndexBatch batch = entityIndex.createBatch();
 
         batch.index( indexSCope, user );
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         final String query = "where username = 'edanuff'";
@@ -550,7 +552,7 @@ public class EntityIndexTest extends BaseIT {
         assertEquals( user.getId(), r.get( 0 ).getId() );
 
         batch.deindex( indexSCope, user.getId(), user.getVersion() );
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         // EntityRef
@@ -611,7 +613,7 @@ public class EntityIndexTest extends BaseIT {
         EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
         batch.index( indexScope, fred);
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         final SearchTypes searchTypes = SearchTypes.fromTypes( "user" );
@@ -688,7 +690,7 @@ public class EntityIndexTest extends BaseIT {
         }
 
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
 
         entityIndex.refreshAsync().toBlocking().first();
 
@@ -755,7 +757,7 @@ public class EntityIndexTest extends BaseIT {
         EntityIndexBatch batch = entityIndex.createBatch();
 
         batch.index( indexSCope, user );
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         final String query = "where searchUUID = " + searchUUID;
@@ -794,7 +796,7 @@ public class EntityIndexTest extends BaseIT {
         EntityIndexBatch batch = entityIndex.createBatch();
 
         batch.index(indexSCope, user);
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
         final String query = "where string = 'I am*'";
@@ -851,7 +853,7 @@ public class EntityIndexTest extends BaseIT {
         EntityIndexBatch batch = entityIndex.createBatch();
         batch.index( indexSCope, first );
         batch.index( indexSCope, second );
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
 
@@ -916,7 +918,7 @@ public class EntityIndexTest extends BaseIT {
         batch.index(indexScope2, second);
 
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
 
@@ -997,7 +999,7 @@ public class EntityIndexTest extends BaseIT {
         batch.index( indexScope2, second);
 
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
 
@@ -1107,7 +1109,7 @@ public class EntityIndexTest extends BaseIT {
         batch.index( indexScope2, second);
 
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
 
 
@@ -1262,7 +1264,7 @@ public class EntityIndexTest extends BaseIT {
         batch.index( indexScope2, second);
 
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
         entityIndex.refreshAsync().toBlocking().first();
         long size = entityIndex.getEntitySize(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE));
         assertTrue( size == 100 );

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
index fbe2bf5..7ecd7a1 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
@@ -82,6 +82,9 @@ public class GeoPagingTest extends BaseIT {
     public CassandraFig cassandraFig;
 
     @Inject
+    public IndexProducer indexProducer;
+
+    @Inject
     @Rule
     public ElasticSearchRule elasticSearchRule;
     private EntityIndex entityIndex;
@@ -128,7 +131,7 @@ public class GeoPagingTest extends BaseIT {
 
         }
 
-        batch.execute().toBlocking().last();
+        indexProducer.put(batch).subscribe();
 
         entityIndex.refreshAsync().toBlocking().last();
 

http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index cca3f0f..c34196e 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -87,6 +87,8 @@ public class IndexLoadTestsIT extends BaseIT {
     @Rule
     public MigrationManagerRule migrationManagerRule;
 
+    @Inject
+    public IndexProducer indexProducer;
 
     @Inject
     public IndexTestFig indexTestFig;
@@ -346,8 +348,8 @@ public class IndexLoadTestsIT extends BaseIT {
                     batchWriteTPS.mark();
 
 
-                    //execute
-                    entityIndexBatch.execute();
+                    indexProducer.put(entityIndexBatch).subscribe();
+
                     //stop
                     time.close();
                 } ).toBlocking().last();