You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/27 21:03:16 UTC

[6/6] git commit: Give two Guice-created objects EntityDeleted and EntityVersionDeletedImpl, access to the Spring-created EntityManagerFactory, also: test fixes.

Give two Guice-created objects EntityDeleted and EntityVersionDeletedImpl, access to the Spring-created EntityManagerFactory, also: test fixes.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/bf347dab
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/bf347dab
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/bf347dab

Branch: refs/heads/two-dot-o-events
Commit: bf347dabfff03a7f81bdb2045fd8ef61bf7a812e
Parents: 50728fe
Author: Dave Johnson <dm...@apigee.com>
Authored: Mon Oct 27 16:02:25 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Mon Oct 27 16:02:25 2014 -0400

----------------------------------------------------------------------
 .../usergrid/corepersistence/CpSetup.java       | 18 +++--
 .../HybridEntityManagerFactory.java             |  4 +
 .../events/EntityDeletedImpl.java               | 14 ++--
 .../events/EntityVersionDeletedImpl.java        | 41 +++++-----
 .../corepersistence/StaleIndexCleanupTest.java  | 82 +++++++++++++++++++-
 5 files changed, 124 insertions(+), 35 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index c1bab12..388bf3b 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -22,7 +22,6 @@ import com.google.inject.Injector;
 import com.netflix.config.ConfigurationManager;
 import java.util.Properties;
 import java.util.UUID;
-import java.util.logging.Level;
 import me.prettyprint.cassandra.service.CassandraHost;
 import me.prettyprint.hector.api.ddl.ComparatorType;
 import static me.prettyprint.hector.api.factory.HFactory.createColumnFamilyDefinition;
@@ -59,7 +58,10 @@ public class CpSetup implements Setup {
 
     private static final Logger logger = LoggerFactory.getLogger( CpSetup.class );
 
-    private final org.apache.usergrid.persistence.EntityManagerFactory emf;
+    private static org.apache.usergrid.persistence.EntityManagerFactory emf;
+
+    private static Injector injector = null;
+
     private final CassandraService cass;
 
     private GuiceModule gm;
@@ -67,16 +69,20 @@ public class CpSetup implements Setup {
 
     /**
      * Instantiates a new setup object.
-     *
-     * @param emf the emf
      */
     public CpSetup( EntityManagerFactory emf, CassandraService cass ) {
-        this.emf = emf;
+        CpSetup.emf = emf;
         this.cass = cass;
     }
 
 
-    private static Injector injector = null;
+    /**
+     * EntityManagerFactory is created by Spring, but Guice-created classes need access to it. 
+     */
+    public static EntityManagerFactory getEntityManagerFactory() {
+        return emf; 
+    }
+
 
     public static Injector getInjector() {
         if ( injector == null ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 54a5dee..6b53d38 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -51,6 +51,10 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
         }
     }
 
+    public EntityManagerFactory getImplementation() {
+        return factory; 
+    }
+
     @Override
     public String getImpementationDescription() throws Exception {
         return factory.getImpementationDescription();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
index 0c49e84..925e01d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityDeletedImpl.java
@@ -26,19 +26,23 @@ import org.apache.usergrid.persistence.model.entity.Id;
 import java.util.UUID;
 import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
 import org.apache.usergrid.corepersistence.CpSetup;
-import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 
 /**
- * purge most current entity
+ * Delete all Query Index indexes associated with an Entity that has just been deleted. 
  */
 public class EntityDeletedImpl implements EntityDeleted {
     private static final Logger logger = LoggerFactory.getLogger( EntityDeletedImpl.class );
 
 
+    public EntityDeletedImpl() {
+        logger.debug("Created");        
+    }
+
     @Override
     public void deleted(CollectionScope scope, Id entityId, UUID version) {
 
@@ -47,10 +51,10 @@ public class EntityDeletedImpl implements EntityDeleted {
             new Object[] { entityId.getType(), entityId.getUuid(), version,
                 scope.getName(), scope.getOwner(), scope.getApplication()});
 
-        CpEntityManagerFactory emf = (CpEntityManagerFactory)
-            CpSetup.getInjector().getInstance( EntityManagerFactory.class );
+        HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+        CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
 
-        final EntityIndex ei = emf.getManagerCache().getEntityIndex(scope);
+        final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
 
         EntityIndexBatch batch = ei.createBatch();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
index 56011a0..fb52573 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/events/EntityVersionDeletedImpl.java
@@ -15,7 +15,6 @@
  * copyright in this work, please see the NOTICE file in the top level
  * directory of this distribution.
  */
-
 package org.apache.usergrid.corepersistence.events;
 
 import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
@@ -37,17 +36,18 @@ import rx.schedulers.Schedulers;
 import java.util.List;
 
 import com.google.inject.Inject;
+import org.apache.usergrid.corepersistence.HybridEntityManagerFactory;
 
 
 /**
  * Purge old entity versions
  */
-public class EntityVersionDeletedImpl implements EntityVersionDeleted{
+public class EntityVersionDeletedImpl implements EntityVersionDeleted {
 
     private final SerializationFig serializationFig;
 
     @Inject
-    public EntityVersionDeletedImpl(SerializationFig fig){
+    public EntityVersionDeletedImpl(SerializationFig fig) {
         this.serializationFig = fig;
     }
 
@@ -55,30 +55,29 @@ public class EntityVersionDeletedImpl implements EntityVersionDeleted{
     public void versionDeleted(
             final CollectionScope scope, final Id entityId, final List<MvccEntity> entityVersions) {
 
-        CpEntityManagerFactory emf = (CpEntityManagerFactory)
-                CpSetup.getInjector().getInstance( EntityManagerFactory.class );
-
-        final EntityIndex ei = emf.getManagerCache().getEntityIndex(scope);
-
-        final EntityIndexBatch entityIndexBatch = ei.createBatch();
+        HybridEntityManagerFactory hemf = (HybridEntityManagerFactory)CpSetup.getEntityManagerFactory();
+        CpEntityManagerFactory cpemf = (CpEntityManagerFactory)hemf.getImplementation();
 
+        final EntityIndex ei = cpemf.getManagerCache().getEntityIndex(scope);
+        
+        final EntityIndexBatch eibatch = ei.createBatch();
 
         final IndexScope indexScope = new IndexScopeImpl(
-                new SimpleId(scope.getOwner().getUuid(),scope.getOwner().getType()),
+                new SimpleId(scope.getOwner().getUuid(), scope.getOwner().getType()),
                 scope.getName()
         );
         rx.Observable.from(entityVersions)
-                .subscribeOn(Schedulers.io())
-                .buffer(serializationFig.getBufferSize())
-                .map(new Func1<List<MvccEntity>,List<MvccEntity>>() {
-                    @Override
-                    public List<MvccEntity> call(List<MvccEntity> entityList) {
-                        for(MvccEntity entity : entityList){
-                             entityIndexBatch.deindex(indexScope,entityId,entity.getVersion());
-                        }
-                        entityIndexBatch.execute();
-                        return entityList;
+            .subscribeOn(Schedulers.io())
+            .buffer(serializationFig.getBufferSize())
+            .map(new Func1<List<MvccEntity>, List<MvccEntity>>() {
+                @Override
+                public List<MvccEntity> call(List<MvccEntity> entityList) {
+                    for (MvccEntity entity : entityList) {
+                        eibatch.deindex(indexScope, entityId, entity.getVersion());
                     }
-                }).toBlocking().last();
+                    eibatch.execute();
+                    return entityList;
+                }
+            }).toBlocking().last();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/bf347dab/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 906d7b8..c4b362d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -95,11 +95,11 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
 
 
     /**
-     * Test that the CpRelationManager cleans up and stale indexes that it finds when it is 
-     * building search results.
+     * Test that the CpRelationManager cleans up stale indexes on read. Ensures that the query 
+     * results builder removes any stale indexes that it finds when building search results.
      */
     @Test
-    public void testStaleIndexCleanup() throws Exception {
+    public void testCleanupOnRead() throws Exception {
 
         logger.info("Started testStaleIndexCleanup()");
 
@@ -202,6 +202,82 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
     }
 
 
+    /**
+     * Test that the EntityDeleteImpl cleans up stale indexes on delete. Ensures that when an 
+     * entity is deleted its old indexes are cleared from ElasticSearch.
+     */
+    @Test
+    public void testCleanupOnDelete() throws Exception {
+
+        logger.info("Started testStaleIndexCleanup()");
+
+        // TODO: turn off index cleanup on read
+
+        final EntityManager em = app.getEntityManager();
+
+        final int numEntities = 10;
+        final int numUpdates = 3;
+
+        // create lots of entities
+        final List<Entity> things = new ArrayList<Entity>(numEntities);
+        for ( int i=0; i<numEntities; i++) {
+            final String thingName = "thing" + i;
+            things.add( em.create("thing", new HashMap<String, Object>() {{
+                put("name", thingName);
+            }}));
+            Thread.sleep( writeDelayMs );
+        }
+        em.refreshIndex();
+
+        CandidateResults crs = queryCollectionCp( "things", "select *");
+        Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
+
+        // update each one a bunch of times
+        int count = 0;
+
+        List<Entity> maxVersions = new ArrayList<>(numEntities);
+
+        for ( Entity thing : things ) {
+            Entity toUpdate = null;
+
+            for ( int j=0; j<numUpdates; j++) {
+                toUpdate = em.get( thing.getUuid() );
+                toUpdate.setProperty( "property"  + j, RandomStringUtils.randomAlphanumeric(10));
+
+                em.update(toUpdate);
+
+                Thread.sleep( writeDelayMs );
+                count++;
+                if ( count % 100 == 0 ) {
+                    logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+                }
+            }
+
+            maxVersions.add( toUpdate );
+        }
+        em.refreshIndex();
+
+        // query Core Persistence directly for total number of result candidates
+        crs = queryCollectionCp("things", "select *");
+        Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+
+        // delete all entities
+        for ( Entity thing : things ) {
+            em.delete( thing );
+        }
+        em.refreshIndex();
+
+        // wait for indexes to be cleared for the deleted entities
+        count = 0;
+        do {
+            Thread.sleep(100);
+            crs = queryCollectionCp("things", "select *");
+        } while ( crs.size() > 0 || count++ > 14 );
+
+        Assert.assertEquals( "Expect no candidates", 0, crs.size() );
+    }
+
+    
     /** 
      * Go around EntityManager and get directly from Core Persistence.
      */