You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/03/18 23:12:42 UTC
[1/3] incubator-usergrid git commit: Fixes issue with repair logic
with incorrect assumptions of version ordering. Versions can appear in any
order during arbitrary order by statements.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o eb0c689da -> d8f949bf0
Fixes issue with repair logic with incorrect assumptions of version ordering. Versions can appear in any order during arbitrary order by statements.
Also fixes overflow logic during repair when you may be more results than anticipated.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/407a4f55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/407a4f55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/407a4f55
Branch: refs/heads/two-dot-o
Commit: 407a4f55eff3adb01f69e7e45f943918e60dd240
Parents: 0334119
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 15:53:43 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 15:53:43 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 ++-
.../results/FilteringLoader.java | 50 +++++++++++++-------
2 files changed, 38 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/407a4f55/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index ee81a67..2eeee28 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -987,7 +987,12 @@ public class CpRelationManager implements RelationManager {
if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
satisfied = true;
}
- else if ( results.size() == originalLimit ) { // got what we need
+
+ /**
+ * In an edge case where we delete stale entities, we could potentially get more results than expected. This will only occur once during the repair phase.
+ * We need to ensure that we short circuit if we overflow the requested limit during a repair.
+ */
+ else if ( results.size() >= originalLimit ) { // got what we need
satisfied = true;
}
else if ( crs.hasCursor() ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/407a4f55/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 2cd9fdc..b571f21 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -143,30 +143,44 @@ public class FilteringLoader implements ResultsLoader {
final UUID currentVersion = currentCandidate.getVersion();
+
+ final CandidateResult toRemove;
+ final CandidateResult toKeep;
+
+ //current is newer than previous. Remove previous and keep current
+ if(UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ){
+ toRemove = previousMax;
+ toKeep = currentCandidate;
+ }
+ //previously seen value is newer than current. Remove the current and keep the previously seen value
+ else{
+ toRemove = currentCandidate;
+ toKeep = previousMax;
+ }
+
//this is a newer version, we know we already have a stale entity, add it to be cleaned up
- if ( UUIDComparator.staticCompare( currentVersion, previousMaxVersion ) > 0 ) {
- //de-index it
- logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
- new Object[] {
- entityId.getUuid(),
- entityId.getType(),
- previousMaxVersion,
- currentVersion } );
- //deindex this document, and remove the previous maxVersion
- //we have to deindex this from our ownerId, since this is what gave us the reference
- indexBatch.deindex( indexScope, previousMax );
- groupedByScopes.remove( collectionType, previousMax );
+ //de-index it
+ logger.warn( "Stale version of Entity uuid:{} type:{}, stale v:{}, latest v:{}",
+ new Object[] {
+ entityId.getUuid(),
+ entityId.getType(),
+ toRemove.getVersion(),
+ toKeep.getVersion() } );
+ //deindex this document, and remove the previous maxVersion
+ //we have to deindex this from our ownerId, since this is what gave us the reference
+ indexBatch.deindex( indexScope, toRemove );
+ groupedByScopes.remove( collectionType, toRemove );
- //TODO, fire the entity repair cleanup task here instead of de-indexing
- //replace the value with a more current version
- maxCandidateMapping.put( entityId, currentCandidate );
- orderIndex.put( entityId, i );
- groupedByScopes.put( collectionType, currentCandidate );
- }
+ //TODO, fire the entity repair cleanup task here instead of de-indexing
+
+ //replace the value with a more current version
+ maxCandidateMapping.put( entityId, toKeep );
+ orderIndex.put( entityId, i );
+ groupedByScopes.put( collectionType, toKeep );
}
[2/3] incubator-usergrid git commit: Added test to prove issue
Posted by sn...@apache.org.
Added test to prove issue
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/143692fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/143692fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/143692fe
Branch: refs/heads/two-dot-o
Commit: 143692fe9e8d06e1a006636b713a68591377fc1d
Parents: 407a4f5
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Mar 18 16:07:46 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Mar 18 16:07:46 2015 -0600
----------------------------------------------------------------------
.../corepersistence/StaleIndexCleanupTest.java | 48 ++++++++++++++++++++
1 file changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/143692fe/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 4bde50e..1becee4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -127,6 +127,54 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
+
+ /**
+ * USERGRID-492 test for ordering
+ */
+ @Test
+ public void testUpdateVersionMaxFirst() throws Exception {
+
+ // turn off post processing stuff that cleans up stale entities
+ System.setProperty( EVENTS_DISABLED, "true" );
+
+ final EntityManager em = app.getEntityManager();
+
+ Entity thing = em.create( "thing", new HashMap<String, Object>() {{
+ put( "ordinal", 0 );
+ }} );
+
+ em.refreshIndex();
+
+ assertEquals( 1, queryCollectionCp( "things", "thing", "select *" ).size() );
+
+ em.updateProperties( thing, new HashMap<String, Object>() {{
+ put( "ordinal", 1 );
+ }} );
+ em.refreshIndex();
+
+ UUID newVersion = getCpEntity( thing ).getVersion();
+
+
+ assertEquals( 2, queryCollectionCp( "things", "thing", "select * order by ordinal desc" ).size() );
+
+ //now run enable events and ensure we clean up
+ System.setProperty( EVENTS_DISABLED, "false" );
+
+ final Results results = queryCollectionEm( "things", "select * order by ordinal desc" );
+
+ assertEquals( 1, results.size() );
+ assertEquals(1, results.getEntities().get( 0 ).getProperty( "ordinal" ));
+
+ em.refreshIndex();
+
+ //ensure it's actually gone
+ final CandidateResults candidates = queryCollectionCp( "things", "thing", "select * order by ordinal desc" );
+ assertEquals( 1, candidates.size() );
+
+ assertEquals(newVersion, candidates.get( 0 ).getVersion());
+ }
+
+
/**
* Test that the CpRelationManager cleans up and stale indexes that it finds when
* it is building search results.
[3/3] incubator-usergrid git commit: Merge branch 'pr/190' into
two-dot-o Fixes issue 492
Posted by sn...@apache.org.
Merge branch 'pr/190' into two-dot-o
Fixes issue 492
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d8f949bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d8f949bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d8f949bf
Branch: refs/heads/two-dot-o
Commit: d8f949bf0baa975fab824779ea8677ecd0b26952
Parents: eb0c689 143692f
Author: Dave Johnson <sn...@apache.org>
Authored: Wed Mar 18 18:11:57 2015 -0400
Committer: Dave Johnson <sn...@apache.org>
Committed: Wed Mar 18 18:11:57 2015 -0400
----------------------------------------------------------------------
.../corepersistence/CpRelationManager.java | 7 ++-
.../results/FilteringLoader.java | 50 +++++++++++++-------
.../corepersistence/StaleIndexCleanupTest.java | 48 +++++++++++++++++++
3 files changed, 86 insertions(+), 19 deletions(-)
----------------------------------------------------------------------