You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/09/21 00:12:42 UTC
[08/21] usergrid git commit: remove batch.execute
remove batch.execute
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/c03a7298
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/c03a7298
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/c03a7298
Branch: refs/heads/two-dot-o-dev
Commit: c03a729817dd7bf5e336a6fe4c9be5984e4b3fb5
Parents: c05d973
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Sep 16 08:21:59 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Sep 16 08:21:59 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 6 +-
.../corepersistence/CpEntityManagerFactory.java | 5 +-
.../corepersistence/CpRelationManager.java | 16 ++---
.../corepersistence/index/IndexServiceImpl.java | 22 ++++---
.../read/search/CandidateEntityFilter.java | 14 +++--
.../pipeline/read/search/CandidateIdFilter.java | 14 +++--
.../persistence/index/EntityIndexBatch.java | 7 ++-
.../index/impl/EsEntityIndexBatchImpl.java | 14 ++---
.../index/impl/EsIndexProducerImpl.java | 9 ++-
.../persistence/index/impl/IndexProducer.java | 2 +
.../persistence/index/impl/EntityIndexTest.java | 66 ++++++++++----------
.../persistence/index/impl/GeoPagingTest.java | 5 +-
.../index/impl/IndexLoadTestsIT.java | 6 +-
13 files changed, 115 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index d46c112..e222f68 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -41,6 +41,7 @@ import org.apache.usergrid.corepersistence.service.ConnectionService;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.IndexLocationStrategy;
import org.apache.usergrid.persistence.index.IndexRefreshCommand;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -183,6 +184,7 @@ public class CpEntityManager implements EntityManager {
private final UUID applicationId;
private final EntityManagerFig entityManagerFig;
+ private final IndexProducer indexProducer;
private Application application;
@@ -246,9 +248,11 @@ public class CpEntityManager implements EntityManager {
final GraphManagerFactory graphManagerFactory,
final CollectionService collectionService,
final ConnectionService connectionService,
+ final IndexProducer indexProducer,
final UUID applicationId ) {
this.entityManagerFig = entityManagerFig;
+ this.indexProducer = indexProducer;
Preconditions.checkNotNull( cass, "cass must not be null" );
Preconditions.checkNotNull( counterUtils, "counterUtils must not be null" );
@@ -759,7 +763,7 @@ public class CpEntityManager implements EntityManager {
Preconditions.checkNotNull(entityRef, "entityRef cannot be null");
CpRelationManager relationManager =
- new CpRelationManager( managerCache, indexService, collectionService, connectionService, this, entityManagerFig, applicationId, entityRef );
+ new CpRelationManager(managerCache, indexService, collectionService, connectionService, indexProducer, this, entityManagerFig, applicationId, entityRef );
return relationManager;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 4bdade5..4a712a7 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -22,6 +22,7 @@ import java.util.Set;
import java.util.TreeMap;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
@@ -96,6 +97,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private static final Logger logger = LoggerFactory.getLogger( CpEntityManagerFactory.class );
private final EntityManagerFig entityManagerFig;
+ private final IndexProducer indexProducer;
private ApplicationContext applicationContext;
@@ -137,6 +139,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.graphManagerFactory = injector.getInstance( GraphManagerFactory.class );
this.collectionService = injector.getInstance( CollectionService.class );
this.connectionService = injector.getInstance( ConnectionService.class );
+ this.indexProducer = injector.getInstance(IndexProducer.class);
//this line always needs to be last due to the temporary cicular dependency until spring is removed
this.applicationIdCache = injector.getInstance(ApplicationIdCacheFactory.class).getInstance(
@@ -199,7 +202,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
private EntityManager _getEntityManager( UUID applicationId ) {
EntityManager em = new CpEntityManager(cassandraService, counterUtils, indexService, managerCache,
- metricsFactory, entityManagerFig, graphManagerFactory, collectionService, connectionService, applicationId );
+ metricsFactory, entityManagerFig, graphManagerFactory, collectionService, connectionService, indexProducer, applicationId );
return em;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 0c3b927..c393fb2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.Assert;
@@ -105,6 +106,7 @@ public class CpRelationManager implements RelationManager {
private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
private final EntityManagerFig entityManagerFig;
+ private final IndexProducer indexProducer;
private ManagerCache managerCache;
@@ -127,9 +129,11 @@ public class CpRelationManager implements RelationManager {
public CpRelationManager( final ManagerCache managerCache,
final AsyncEventService indexService, final CollectionService collectionService,
- final ConnectionService connectionService, final EntityManager em,
+ final ConnectionService connectionService,final IndexProducer indexProducer,
+ final EntityManager em,
final EntityManagerFig entityManagerFig, final UUID applicationId,
- final EntityRef headEntity ) {
+ final EntityRef headEntity) {
+ this.indexProducer = indexProducer;
Assert.notNull( em, "Entity manager cannot be null" );
@@ -529,16 +533,14 @@ public class CpRelationManager implements RelationManager {
*/
final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
- final EntityIndexBatch batch = ei.createBatch();
// remove item from collection index
SearchEdge indexScope = createCollectionSearchEdge( cpHeadEntity.getId(), collectionName );
+ final EntityIndexBatch batch = ei.createBatch();
batch.deindex( indexScope, memberEntity );
-
-
- batch.execute();
-
+ //TODO: this should not happen here, needs to go to SQS
+ indexProducer.put(batch).subscribe();
// special handling for roles collection of a group
if ( headEntity.getType().equals( Group.ENTITY_TYPE ) ) {
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
index 8a8dba1..be7dc20 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/IndexServiceImpl.java
@@ -24,6 +24,7 @@ import java.util.Iterator;
import java.util.UUID;
import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.utils.UUIDUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -68,18 +69,25 @@ public class IndexServiceImpl implements IndexService {
private final EdgesObservable edgesObservable;
private final IndexFig indexFig;
private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+ private final IndexProducer indexProducer;
private final Timer indexTimer;
private final Timer addTimer;
@Inject
- public IndexServiceImpl( final GraphManagerFactory graphManagerFactory, final EntityIndexFactory entityIndexFactory,
- final EdgesObservable edgesObservable, final IndexFig indexFig, final IndexLocationStrategyFactory indexLocationStrategyFactory, final MetricsFactory metricsFactory ) {
+ public IndexServiceImpl( final GraphManagerFactory graphManagerFactory,
+ final EntityIndexFactory entityIndexFactory,
+ final EdgesObservable edgesObservable,
+ final IndexFig indexFig,
+ final IndexLocationStrategyFactory indexLocationStrategyFactory,
+ final MetricsFactory metricsFactory,
+ final IndexProducer indexProducer) {
this.graphManagerFactory = graphManagerFactory;
this.entityIndexFactory = entityIndexFactory;
this.edgesObservable = edgesObservable;
this.indexFig = indexFig;
this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+ this.indexProducer = indexProducer;
this.indexTimer = metricsFactory.getTimer( IndexServiceImpl.class, "index.update_all");
this.addTimer = metricsFactory.getTimer( IndexServiceImpl.class, "index.add" );
}
@@ -89,7 +97,7 @@ public class IndexServiceImpl implements IndexService {
public Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope,
final Entity entity ) {
//bootstrap the lower modules from their caches
- final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope );
+ final GraphManager gm = graphManagerFactory.createEdgeManager(applicationScope);
final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope));
@@ -114,7 +122,7 @@ public class IndexServiceImpl implements IndexService {
batch.index( indexEdge, entity );
} )
//return the future from the batch execution
- .flatMap( batch -> batch.execute() ) );
+ .flatMap( batch -> indexProducer.put(batch) ) );
return ObservableTimer.time( batches, indexTimer );
}
@@ -142,7 +150,7 @@ public class IndexServiceImpl implements IndexService {
batch.index( indexEdge, entity );
- return batch.execute();
+ return indexProducer.put(batch);
} );
return ObservableTimer.time( batches, addTimer );
@@ -185,7 +193,7 @@ public class IndexServiceImpl implements IndexService {
batch = deindexBatchIteratorResolver( fromTarget, sourceEdgesToBeDeindexed, batch );
- return batch.execute();
+ return indexProducer.put(batch);
} );
return ObservableTimer.time( batches, addTimer );
@@ -221,7 +229,7 @@ public class IndexServiceImpl implements IndexService {
batch.deindex( searchEdge, candidateResult );
} )
//return the future from the batch execution
- .flatMap( batch -> batch.execute() );
+ .flatMap( batch -> indexProducer.put(batch) );
return ObservableTimer.time(batches, indexTimer);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
index 14c880f..ac2962a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateEntityFilter.java
@@ -24,6 +24,7 @@ import java.util.*;
import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.apache.usergrid.persistence.model.field.Field;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,16 +57,19 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private final EntityIndexFactory entityIndexFactory;
private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+ private final IndexProducer producer;
@Inject
public CandidateEntityFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
final EntityIndexFactory entityIndexFactory,
- final IndexLocationStrategyFactory indexLocationStrategyFactory
+ final IndexLocationStrategyFactory indexLocationStrategyFactory,
+ final IndexProducer producer
) {
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
this.entityIndexFactory = entityIndexFactory;
this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+ this.producer = producer;
}
@@ -107,7 +111,7 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
.flatMap(idList -> entityCollectionManager.load(idList));
//now we have a collection, validate our canidate set is correct.
return entitySets.map(
- entitySet -> new EntityVerifier(
+ entitySet -> new EntityVerifier(producer,
applicationIndex.createBatch(), entitySet, candidateResults)
)
.doOnNext(entityCollector -> entityCollector.merge())
@@ -148,13 +152,15 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
private static final Logger logger = LoggerFactory.getLogger( EntityVerifier.class );
private List<FilterResult<Entity>> results = new ArrayList<>();
+ private final IndexProducer producer;
private final EntityIndexBatch batch;
private final List<FilterResult<Candidate>> candidateResults;
private final EntitySet entitySet;
- public EntityVerifier( final EntityIndexBatch batch, final EntitySet entitySet,
+ public EntityVerifier( final IndexProducer producer, final EntityIndexBatch batch, final EntitySet entitySet,
final List<FilterResult<Candidate>> candidateResults ) {
+ this.producer = producer;
this.batch = batch;
this.entitySet = entitySet;
this.candidateResults = candidateResults;
@@ -171,7 +177,7 @@ public class CandidateEntityFilter extends AbstractFilter<FilterResult<Candidate
validate( candidateResult );
}
- batch.execute().subscribe();
+ producer.put(batch).subscribe();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
index 3b1c102..b00efee 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/pipeline/read/search/CandidateIdFilter.java
@@ -26,6 +26,7 @@ import java.util.UUID;
import org.apache.usergrid.corepersistence.index.IndexLocationStrategyFactory;
import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.impl.IndexProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,15 +54,18 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
private final EntityCollectionManagerFactory entityCollectionManagerFactory;
private final EntityIndexFactory entityIndexFactory;
private final IndexLocationStrategyFactory indexLocationStrategyFactory;
+ private final IndexProducer indexProducer;
@Inject
public CandidateIdFilter( final EntityCollectionManagerFactory entityCollectionManagerFactory,
final EntityIndexFactory entityIndexFactory,
- final IndexLocationStrategyFactory indexLocationStrategyFactory) {
+ final IndexLocationStrategyFactory indexLocationStrategyFactory,
+ final IndexProducer indexProducer) {
this.entityCollectionManagerFactory = entityCollectionManagerFactory;
this.entityIndexFactory = entityIndexFactory;
this.indexLocationStrategyFactory = indexLocationStrategyFactory;
+ this.indexProducer = indexProducer;
}
@@ -96,7 +100,7 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
//now we have a collection, validate our canidate set is correct.
return versionSetObservable.map(
- entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet,
+ entitySet -> new EntityCollector(indexProducer, applicationIndex.createBatch(), entitySet,
candidateResults ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap(
entityCollector -> Observable.from( entityCollector.collectResults() ) );
} );
@@ -113,13 +117,15 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
private static final Logger logger = LoggerFactory.getLogger( EntityCollector.class );
private List<FilterResult<Id>> results = new ArrayList<>();
+ private final IndexProducer indexProducer;
private final EntityIndexBatch batch;
private final List<FilterResult<Candidate>> candidateResults;
private final VersionSet versionSet;
- public EntityCollector( final EntityIndexBatch batch, final VersionSet versionSet,
+ public EntityCollector( final IndexProducer indexProducer, final EntityIndexBatch batch, final VersionSet versionSet,
final List<FilterResult<Candidate>> candidateResults ) {
+ this.indexProducer = indexProducer;
this.batch = batch;
this.versionSet = versionSet;
this.candidateResults = candidateResults;
@@ -136,7 +142,7 @@ public class CandidateIdFilter extends AbstractFilter<FilterResult<Candidate>, F
validate( candidateResult );
}
- batch.execute();
+ indexProducer.put(batch).subscribe();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
index d1b076d..baca504 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndexBatch.java
@@ -63,10 +63,11 @@ public interface EntityIndexBatch {
/**
- * Execute the batch
- * @return future to guarantee execution
+ * return built payload
+ * @return
*/
- Observable<IndexOperationMessage> execute();
+ IndexOperationMessage build();
+
/**
* Get the number of operations in the batch
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 64a1c6a..3679887 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -66,8 +66,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity ) {
- IndexValidationUtils.validateIndexEdge( indexEdge );
- ValidationUtils.verifyEntityWrite( entity );
+ IndexValidationUtils.validateIndexEdge(indexEdge);
+ ValidationUtils.verifyEntityWrite(entity);
ValidationUtils.verifyVersion( entity.getVersion() );
final String writeAlias = alias.getWriteAlias();
@@ -78,7 +78,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
//add app id for indexing
- container.addIndexRequest( new IndexOperation( writeAlias, applicationScope, indexEdge, entity ) );
+ container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity));
return this;
}
@@ -115,18 +115,16 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@Override
- public EntityIndexBatch deindex( final SearchEdge searchEdge, final CandidateResult entity ) {
+ public EntityIndexBatch deindex(final SearchEdge searchEdge, final CandidateResult entity) {
return deindex( searchEdge, entity.getId(), entity.getVersion() );
}
-
@Override
- public Observable execute() {
- return indexBatchBufferProducer.put( container );
+ public IndexOperationMessage build(){
+ return container;
}
-
@Override
public int size() {
return container.getDeIndexRequests().size() + container.getIndexRequests().size();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index efce823..ef59abb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -24,6 +24,7 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import com.codahale.metrics.Histogram;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.elasticsearch.action.WriteConsistencyLevel;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
@@ -87,8 +88,14 @@ public class EsIndexProducerImpl implements IndexProducer {
}
+ public Observable<IndexOperationMessage> put( EntityIndexBatch batch ) {
+ Preconditions.checkNotNull(batch, "Batch cannot be null");
+ return put(batch.build());
+ }
+
public Observable<IndexOperationMessage> put( IndexOperationMessage message ) {
- Preconditions.checkNotNull(message, "Message cannot be null");
+
+ Preconditions.checkNotNull(message, "Message cannot be null");
indexSizeCounter.inc(message.getDeIndexRequests().size());
indexSizeCounter.inc(message.getIndexRequests().size());
return processBatch(message);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
index e18ac50..8f3ab79 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexProducer.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.index.impl;
+import org.apache.usergrid.persistence.index.EntityIndexBatch;
import rx.Observable;
@@ -34,6 +35,7 @@ public interface IndexProducer {
* @param message
* @return
*/
+ Observable<IndexOperationMessage> put(EntityIndexBatch message);
Observable<IndexOperationMessage> put(IndexOperationMessage message);
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 6348a44..2fd9d3b 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -77,6 +77,8 @@ public class EntityIndexTest extends BaseIT {
@Inject
public EntityIndexFactory eif;
+ @Inject
+ public IndexProducer indexProducer;
@Inject
public IndexFig fig;
@@ -133,25 +135,25 @@ public class EntityIndexTest extends BaseIT {
entity1.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID()));
entity1.setField( new StringField( "testfield", "test" ) );
entity1.setField(new IntegerField("ordinal", 0));
- entity1.setField(new UUIDField("testuuid",uuid));
+ entity1.setField(new UUIDField("testuuid", uuid));
- batch.index( indexEdge, entity1 );
- batch.execute().toBlocking().last();
+ batch.index(indexEdge, entity1);
+ indexProducer.put(batch).subscribe();
Entity entity2 = new Entity( entityType );
- EntityUtils.setVersion( entity2, UUIDGenerator.newTimeUUID() );
+ EntityUtils.setVersion(entity2, UUIDGenerator.newTimeUUID());
List<String> list = new ArrayList<>();
- list.add( "test" );
- entity2.setField( new ArrayField<>( "testfield", list ) );
- entity2.setField( new IntegerField( "ordinal", 1 ) );
+ list.add("test");
+ entity2.setField(new ArrayField<>("testfield", list));
+ entity2.setField(new IntegerField("ordinal", 1));
- batch.index( indexEdge, entity2 );
- batch.execute().toBlocking().last();
+ batch.index(indexEdge, entity2);
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
@@ -163,14 +165,14 @@ public class EntityIndexTest extends BaseIT {
timer.stop();
- assertEquals( 2, candidateResults.size() );
- log.debug( "Query time {}ms", timer.getTime() );
+ assertEquals(2, candidateResults.size());
+ log.debug("Query time {}ms", timer.getTime());
final CandidateResult candidate1 = candidateResults.get(0);
//check the id and version
assertEquals( entity1.getId(), candidate1.getId() );
- assertEquals( entity1.getVersion(), candidate1.getVersion() );
+ assertEquals(entity1.getVersion(), candidate1.getVersion());
final CandidateResult candidate2 = candidateResults.get(1);
@@ -215,7 +217,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
insertJsonBlob( sampleJson, batch, entityType, indexEdge, size, 0 );
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
}
catch ( Exception e ) {
synchronized ( failTime ) {
@@ -290,7 +292,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch entityIndexBatch = entityIndex.createBatch();
entityIndexBatch.deindex(searchEdge, crs.get(0));
- entityIndexBatch.execute().toBlocking().last();
+ indexProducer.put(entityIndex.createBatch() ).subscribe();
entityIndex.refreshAsync().toBlocking().first();
//Hilda Youn
@@ -306,7 +308,7 @@ public class EntityIndexTest extends BaseIT {
});
EntityIndexBatch batch = entityIndex.createBatch();
insertJsonBlob(sampleJson, batch, entityType, indexEdge, max, startIndex);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
IndexRefreshCommandImpl.IndexRefreshCommandInfo info = entityIndex.refreshAsync().toBlocking().first();
long time = info.getExecutionTime();
log.info("refresh took ms:" + time);
@@ -360,11 +362,11 @@ public class EntityIndexTest extends BaseIT {
Entity entity = EntityIndexMapUtils.fromMap( entityMap );
- EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
- EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
- entity.setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
+ EntityUtils.setId(entity, new SimpleId( "fastcar" ) );
+ EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() );
+ entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) );
- entityIndex.createBatch().index( searchEdge, entity ).execute().toBlocking().last();
+ indexProducer.put(entityIndex.createBatch().index( searchEdge, entity )).toBlocking().last();
entityIndex.refreshAsync().toBlocking().first();
CandidateResults candidateResults = entityIndex
@@ -373,7 +375,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
batch.deindex( searchEdge, entity );
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
candidateResults = entityIndex
@@ -410,7 +412,7 @@ public class EntityIndexTest extends BaseIT {
entity[i].setField( new UUIDField( IndexingUtils.ENTITY_ID_FIELDNAME, entityUUID ) );
//index the new entity. This is where the loop will be set to create like 100 entities.
- entityIndex.createBatch().index( searchEdge, entity[i] ).execute().toBlocking().last();
+ indexProducer.put(entityIndex.createBatch().index( searchEdge, entity[i] )).toBlocking().last();
}
entityIndex.refreshAsync().toBlocking().first();
@@ -541,7 +543,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
batch.index( indexSCope, user );
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
final String query = "where username = 'edanuff'";
@@ -550,7 +552,7 @@ public class EntityIndexTest extends BaseIT {
assertEquals( user.getId(), r.get( 0 ).getId() );
batch.deindex( indexSCope, user.getId(), user.getVersion() );
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
// EntityRef
@@ -611,7 +613,7 @@ public class EntityIndexTest extends BaseIT {
EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
batch.index( indexScope, fred);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
final SearchTypes searchTypes = SearchTypes.fromTypes( "user" );
@@ -688,7 +690,7 @@ public class EntityIndexTest extends BaseIT {
}
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
@@ -755,7 +757,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
batch.index( indexSCope, user );
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
final String query = "where searchUUID = " + searchUUID;
@@ -794,7 +796,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
batch.index(indexSCope, user);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
final String query = "where string = 'I am*'";
@@ -851,7 +853,7 @@ public class EntityIndexTest extends BaseIT {
EntityIndexBatch batch = entityIndex.createBatch();
batch.index( indexSCope, first );
batch.index( indexSCope, second );
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
@@ -916,7 +918,7 @@ public class EntityIndexTest extends BaseIT {
batch.index(indexScope2, second);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
@@ -997,7 +999,7 @@ public class EntityIndexTest extends BaseIT {
batch.index( indexScope2, second);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
@@ -1107,7 +1109,7 @@ public class EntityIndexTest extends BaseIT {
batch.index( indexScope2, second);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
@@ -1262,7 +1264,7 @@ public class EntityIndexTest extends BaseIT {
batch.index( indexScope2, second);
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().first();
long size = entityIndex.getEntitySize(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE));
assertTrue( size == 100 );
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
index fbe2bf5..7ecd7a1 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/GeoPagingTest.java
@@ -82,6 +82,9 @@ public class GeoPagingTest extends BaseIT {
public CassandraFig cassandraFig;
@Inject
+ public IndexProducer indexProducer;
+
+ @Inject
@Rule
public ElasticSearchRule elasticSearchRule;
private EntityIndex entityIndex;
@@ -128,7 +131,7 @@ public class GeoPagingTest extends BaseIT {
}
- batch.execute().toBlocking().last();
+ indexProducer.put(batch).subscribe();
entityIndex.refreshAsync().toBlocking().last();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/c03a7298/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index cca3f0f..c34196e 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -87,6 +87,8 @@ public class IndexLoadTestsIT extends BaseIT {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Inject
+ public IndexProducer indexProducer;
@Inject
public IndexTestFig indexTestFig;
@@ -346,8 +348,8 @@ public class IndexLoadTestsIT extends BaseIT {
batchWriteTPS.mark();
- //execute
- entityIndexBatch.execute();
+ indexProducer.put(entityIndexBatch).subscribe();
+
//stop
time.close();
} ).toBlocking().last();