You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/10/10 18:55:33 UTC
git commit: Fixes cleanup by moving de-index event to post complete
results for consistent views.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o befcdcab6 -> cf0c4b84b
Fixes cleanup by moving de-index event to post complete results for consistent views.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cf0c4b84
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cf0c4b84
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cf0c4b84
Branch: refs/heads/two-dot-o
Commit: cf0c4b84bf0320654bb14208da1c29fe13d9ad31
Parents: befcdca
Author: Todd Nine <to...@apache.org>
Authored: Fri Oct 10 10:55:19 2014 -0600
Committer: Todd Nine <to...@apache.org>
Committed: Fri Oct 10 10:55:19 2014 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 14 +++++-
.../results/FilteringLoader.java | 52 ++++++++++++--------
.../corepersistence/results/ResultsLoader.java | 11 +++--
.../corepersistence/StaleIndexCleanupTest.java | 2 +
4 files changed, 53 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf0c4b84/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 76b8930..bac085c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -32,6 +32,7 @@ import java.util.Map;
import java.util.Set;
import java.util.UUID;
+import org.apache.usergrid.corepersistence.results.ResultsLoader;
import org.apache.usergrid.corepersistence.results.ResultsLoaderFactory;
import org.apache.usergrid.corepersistence.results.ResultsLoaderFactoryImpl;
import org.apache.usergrid.corepersistence.util.CpEntityMapUtils;
@@ -1553,8 +1554,17 @@ public class CpRelationManager implements RelationManager {
logger.debug("buildResults() for {} from {} candidates", collName, crs.size());
- final Results results = this.resultsLoaderFactory.getLoader(
- applicationScope, this.headEntity, query.getResultsLevel() ).getResults( crs );
+ //get an instance of our results loader
+ final ResultsLoader resultsLoader = this.resultsLoaderFactory.getLoader( applicationScope, this.headEntity,
+ query.getResultsLevel() );
+
+ //load the results
+ final Results results = resultsLoader.loadResults( crs );
+
+ //signal for post processing
+ resultsLoader.postProcess();
+
+
results.setCursor( crs.getCursor() );
results.setQueryProcessor( new CpQueryProcessor(em, query, headEntity, collName) );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf0c4b84/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 9e5ad55..4973783 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -64,6 +64,7 @@ public class FilteringLoader implements ResultsLoader {
private final ResultsVerifier resultsVerifier;
private final Id ownerId;
private final ApplicationScope applicationScope;
+ private final EntityIndexBatch indexBatch;
protected FilteringLoader( final CpManagerCache managerCache, final ResultsVerifier resultsVerifier,
@@ -72,16 +73,18 @@ public class FilteringLoader implements ResultsLoader {
this.resultsVerifier = resultsVerifier;
this.ownerId = new SimpleId( ownerId.getUuid(), ownerId.getType() );
this.applicationScope = applicationScope;
+
+ final EntityIndex index = managerCache.getEntityIndex( applicationScope );
+
+ indexBatch = index.createBatch();
}
@Override
- public Results getResults( final CandidateResults crs ) {
+ public Results loadResults( final CandidateResults crs ) {
- final EntityIndex index = managerCache.getEntityIndex( applicationScope );
- final EntityIndexBatch indexBatch = index.createBatch();
/**
* For each entity, holds the index it appears in our candidates for keeping ordering correct
@@ -98,21 +101,25 @@ public class FilteringLoader implements ResultsLoader {
* so we want to batch
* fetch them more efficiently
*/
- final HashMultimap<String, CandidateResult> groupedByScopes =
- HashMultimap.create( crs.size(), crs.size() );
+ final HashMultimap<String, CandidateResult> groupedByScopes = HashMultimap.create( crs.size(), crs.size() );
final Iterator<CandidateResult> iter = crs.iterator();
/**
+ * TODO, in this case we're "optimizing" due to the limitations of collection scope. Perhaps we should
+ * change the API to just be an application, then an "owner" scope?
+ */
+
+ /**
* Go through the candidates and group them by scope for more efficient retrieval
*/
for ( int i = 0; iter.hasNext(); i++ ) {
final CandidateResult currentCandidate = iter.next();
- final String collectionType = CpNamingUtils.getCollectionScopeNameFromEntityType(
- currentCandidate.getId().getType() );
+ final String collectionType =
+ CpNamingUtils.getCollectionScopeNameFromEntityType( currentCandidate.getId().getType() );
final Id entityId = currentCandidate.getId();
@@ -137,10 +144,9 @@ public class FilteringLoader implements ResultsLoader {
if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
//de-index it
- logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] {
- entityId.getUuid(), entityId.getType(), previousMaxVersion, currentVersion
- } );
+ logger.debug( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}", new Object[] {
+ entityId.getUuid(), entityId.getType(), previousMaxVersion, currentVersion
+ } );
//deindex this document, and remove the previous maxVersion
deIndex( indexBatch, ownerId, previousMax );
@@ -183,15 +189,14 @@ public class FilteringLoader implements ResultsLoader {
//now using the scope, load the collection
-
+
// Get the collection scope and batch load all the versions
- final CollectionScope collScope = new CollectionScopeImpl(
- applicationScope.getApplication(),
- applicationScope.getApplication(),
- scopeName );
+ final CollectionScope collScope =
+ new CollectionScopeImpl( applicationScope.getApplication(), applicationScope.getApplication(),
+ scopeName );
- final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope);
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collScope );
//load the results into the loader for this scope for validation
@@ -218,17 +223,22 @@ public class FilteringLoader implements ResultsLoader {
//execute the cleanup
- indexBatch.execute();
+// indexBatch.execute();
return resultsVerifier.getResults( sortedResults.values() );
}
- protected void deIndex( final EntityIndexBatch batch, final Id ownerId,
- final CandidateResult candidateResult ) {
+ @Override
+ public void postProcess() {
+ this.indexBatch.execute();
+ }
+
+
+ protected void deIndex( final EntityIndexBatch batch, final Id ownerId, final CandidateResult candidateResult ) {
IndexScope indexScope = new IndexScopeImpl( ownerId,
- CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType() ) );
batch.deindex( indexScope, candidateResult );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf0c4b84/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
index 648ee0c..28df6b6 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoader.java
@@ -30,9 +30,14 @@ import org.apache.usergrid.persistence.index.query.CandidateResults;
public interface ResultsLoader {
/**
- * Using the candidate results, get the results
+ * Using the candidate results, load our results. Should filter stale results
* @param crs The candidate result set
- * @return
+ * @return Results. Null safe, but may be empty
*/
- public Results getResults( final CandidateResults crs);
+ public Results loadResults( final CandidateResults crs);
+
+ /**
+ * Post process the load operation
+ */
+ public void postProcess();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cf0c4b84/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 159144d..906d7b8 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -190,6 +190,8 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
assertEquals( "Expect no stale candidates", numEntities, thingCount );
+ em.refreshIndex();
+
// EntityManager should have kicked off a batch cleanup of those stale indexes
// wait a second for batch cleanup to complete
Thread.sleep(600);