You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/27 21:03:16 UTC
[6/6] git commit: Give two Guice-created objects EntityDeleted and
EntityVersionDeletedImpl, access to the Spring-created EntityManagerFactory,
also: test fixes.
Give two Guice-created objects EntityDeleted and EntityVersionDeletedImpl, access to the Spring-created EntityManagerFactory, also: test fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bf347dab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bf347dab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bf347dab
Branch: refs/heads/two-dot-o-events
Commit: bf347dabfff03a7f81bdb2045fd8ef61bf7a812e
Parents: 50728fe
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Oct 27 16:02:25 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Oct 27 16:02:25 2014 -0400
----------------------------------------------------------------------
.../usergrid/corepersistence/CpSetup.java | 18 +++--
.../HybridEntityManagerFactory.java | 4 +
.../events/EntityDeletedImpl.java | 14 ++--
.../events/EntityVersionDeletedImpl.java | 41 +++++-----
.../corepersistence/StaleIndexCleanupTest.java | 82 +++++++++++++++++++-
5 files changed, 124 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index c1bab12..388bf3b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -22,7 +22,6 @@ import com.google.inject.Injector;
import com.netflix.config.ConfigurationManager;
import java.util.Properties;
import java.util.UUID;
-import java.util.logging.Level;
import me.prettyprint.cassandra.service.CassandraHost;
import me.prettyprint.hector.api.ddl.ComparatorType;
import static me.prettyprint.hector.api.factory.HFactory.createColumnFamilyDefinition;
@@ -59,7 +58,10 @@ public class CpSetup implements Setup {
private static final Logger logger = LoggerFactory.getLogger( CpSetup.class );
- private final org.apache.usergrid.persistence.EntityManagerFactory emf;
+ private static org.apache.usergrid.persistence.EntityManagerFactory emf;
+
+ private static Injector injector = null;
+
private final CassandraService cass;
private GuiceModule gm;
@@ -67,16 +69,20 @@ public class CpSetup implements Setup {
/**
* Instantiates a new setup object.
- *
- * @param emf the emf
*/
public CpSetup( EntityManagerFactory emf, CassandraService cass ) {
- this.emf = emf;
+ CpSetup.emf = emf;
this.cass = cass;
}
- private static Injector injector = null;
+ /**
+ * EntityManagerFactory is created by Spring, but Guice-created classes need access to it.
+ */
+ public static EntityManagerFactory getEntityManagerFactory() {
+ return emf;
+ }
+
public static Injector getInjector() {
if ( injector == null ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 54a5dee..6b53d38 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -51,6 +51,10 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
}
}
+ public EntityManagerFactory getImplementation() {
+ return factory;
+ }
+
@Override
public String getImpementationDescription() throws Exception {
return factory.getImpementationDescription();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
index 0c49e84..925e01d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
@@ -26,19 +26,23 @@ import org.apache.usergrid.persistence.model.entity.Id;
import java.util.UUID;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
import org.apache.usergrid.corepersistence.CpSetup;
-import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
import org.apache.usergrid.persistence.index.EntityIndex;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * purge most current entity
+ * Delete all Query Index indexes associated with an Entity that has just been deleted.
*/
public class EntityDeletedImpl implements EntityDeleted {
private static final Logger logger = LoggerFactory.getLogger( EntityDeletedImpl.class );
+ public EntityDeletedImpl() {
+ logger.debug("Created");
+ }
+
@Override
public void deleted(CollectionScope scope, Id entityId, UUID version) {
@@ -47,10 +51,10 @@ public class EntityDeletedImpl implements EntityDeleted {
new Object[] { entityId.getType(), entityId.getUuid(), version,
scope.getName(), scope.getOwner(), scope.getApplication()});
- CpEntityManagerFactory emf = (CpEntityManagerFactory)
- CpSetup.getInjector().getInstance( EntityManagerFactory.class );
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
- final EntityIndex ei = emf.getManagerCache().getEntityIndex(scope);
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
EntityIndexBatch batch = ei.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
index 56011a0..fb52573 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
@@ -15,7 +15,6 @@
* copyright in this work, please see the NOTICE file in the top level
* directory of this distribution.
*/
-
package org.apache.usergrid.corepersistence.events;
import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
@@ -37,17 +36,18 @@ import rx.schedulers.Schedulers;
import java.util.List;
import com.google.inject.Inject;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
/**
* Purge old entity versions
*/
-public class EntityVersionDeletedImpl implements EntityVersionDeleted{
+public class EntityVersionDeletedImpl implements EntityVersionDeleted {
private final SerializationFig serializationFig;
@Inject
- public EntityVersionDeletedImpl(SerializationFig fig){
+ public EntityVersionDeletedImpl(SerializationFig fig) {
this.serializationFig = fig;
}
@@ -55,30 +55,29 @@ public class EntityVersionDeletedImpl implements EntityVersionDeleted{
public void versionDeleted(
final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
- CpEntityManagerFactory emf = (CpEntityManagerFactory)
- CpSetup.getInjector().getInstance( EntityManagerFactory.class );
-
- final EntityIndex ei = emf.getManagerCache().getEntityIndex(scope);
-
- final EntityIndexBatch entityIndexBatch = ei.createBatch();
+ HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+ CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
+ final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+
+ final EntityIndexBatch eibatch = ei.createBatch();
final IndexScope indexScope = new IndexScopeImpl(
- new SimpleId(scope.getOwner().getUuid(),scope.getOwner().getType()),
+ new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
scope.getName()
);
rx.Observable.from(entityVersions)
- .subscribeOn(Schedulers.io())
- .buffer(serializationFig.getBufferSize())
- .map(new Func1<List<MvccEntity>,List<MvccEntity>>() {
- @Override
- public List<MvccEntity> call(List<MvccEntity> entityList) {
- for(MvccEntity entity : entityList){
- entityIndexBatch.deindex(indexScope,entityId,entity.getVersion());
- }
- entityIndexBatch.execute();
- return entityList;
+ .subscribeOn(Schedulers.io())
+ .buffer(serializationFig.getBufferSize())
+ .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+ @Override
+ public List<MvccEntity> call(List<MvccEntity> entityList) {
+ for (MvccEntity entity : entityList) {
+ eibatch.deindex(indexScope, entityId, entity.getVersion());
}
- }).toBlocking().last();
+ eibatch.execute();
+ return entityList;
+ }
+ }).toBlocking().last();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 906d7b8..c4b362d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -95,11 +95,11 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
/**
- * Test that the CpRelationManager cleans up and stale indexes that it finds when it is
- * building search results.
+ * Test that the CpRelationManager cleans up stale indexes on read. Ensures that the query
+ * results builder removes any stale indexes that it finds when building search results.
*/
@Test
- public void testStaleIndexCleanup() throws Exception {
+ public void testCleanupOnRead() throws Exception {
logger.info("Started testStaleIndexCleanup()");
@@ -202,6 +202,82 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
}
+ /**
+ * Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an
+ * entity is deleted its old indexes are cleared from ElasticSearch.
+ */
+ @Test
+ public void testCleanupOnDelete() throws Exception {
+
+ logger.info("Started testStaleIndexCleanup()");
+
+ // TODO: turn off index cleanup on read
+
+ final EntityManager em = app.getEntityManager();
+
+ final int numEntities = 10;
+ final int numUpdates = 3;
+
+ // create lots of entities
+ final List<Entity> things = new ArrayList<Entity>(numEntities);
+ for ( int i=0; i<numEntities; i++) {
+ final String thingName = "thing" + i;
+ things.add( em.create("thing", new HashMap<String, Object>() {{
+ put("name", thingName);
+ }}));
+ Thread.sleep( writeDelayMs );
+ }
+ em.refreshIndex();
+
+ CandidateResults crs = queryCollectionCp( "things", "select *");
+ Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
+
+ // update each one a bunch of times
+ int count = 0;
+
+ List<Entity> maxVersions = new ArrayList<>(numEntities);
+
+ for ( Entity thing : things ) {
+ Entity toUpdate = null;
+
+ for ( int j=0; j<numUpdates; j++) {
+ toUpdate = em.get( thing.getUuid() );
+ toUpdate.setProperty( "property" + j, RandomStringUtils.randomAlphanumeric(10));
+
+ em.update(toUpdate);
+
+ Thread.sleep( writeDelayMs );
+ count++;
+ if ( count % 100 == 0 ) {
+ logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+ }
+ }
+
+ maxVersions.add( toUpdate );
+ }
+ em.refreshIndex();
+
+ // query Core Persistence directly for total number of result candidates
+ crs = queryCollectionCp("things", "select *");
+ Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+
+ // delete all entities
+ for ( Entity thing : things ) {
+ em.delete( thing );
+ }
+ em.refreshIndex();
+
+ // wait for indexes to be cleared for the deleted entities
+ count = 0;
+ do {
+ Thread.sleep(100);
+ crs = queryCollectionCp("things", "select *");
+ } while ( crs.size() > 0 || count++ > 14 );
+
+ Assert.assertEquals( "Expect no candidates", 0, crs.size() );
+ }
+
+
/**
* Go around EntityManager and get directly from Core Persistence.
*/