You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/02 06:12:12 UTC

[03/12] Changes to 1) ensure that the CpEntityManager's ElasticSearch can be rebuilt entirely from data stored in Cassandra, 2) provide support Index Rebuild in two-dot-o and the beginnings of an Index Rebuild test.

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 0345e23..4c60e82 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -16,9 +16,11 @@
 
 package org.apache.usergrid.corepersistence;
 
+import com.yammer.metrics.annotation.Metered;
 import java.nio.ByteBuffer;
 import java.util.AbstractMap;
 import java.util.ArrayList;
+import static java.util.Arrays.asList;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -27,12 +29,13 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
-
-import org.apache.usergrid.utils.UUIDUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.util.Assert;
-
+import me.prettyprint.hector.api.Keyspace;
+import me.prettyprint.hector.api.beans.DynamicComposite;
+import me.prettyprint.hector.api.beans.HColumn;
+import static me.prettyprint.hector.api.factory.HFactory.createMutator;
+import me.prettyprint.hector.api.mutation.Mutator;
+import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromCollectionName;
+import static org.apache.usergrid.corepersistence.CpEntityManager.getEdgeTypeFromConnectionType;
 import org.apache.usergrid.persistence.ConnectedEntityRef;
 import org.apache.usergrid.persistence.ConnectionRef;
 import org.apache.usergrid.persistence.Entity;
@@ -45,12 +48,43 @@ import org.apache.usergrid.persistence.RelationManager;
 import org.apache.usergrid.persistence.Results;
 import org.apache.usergrid.persistence.RoleRef;
 import org.apache.usergrid.persistence.Schema;
+import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES;
+import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_TYPES;
+import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
+import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE;
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
+import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
+import static org.apache.usergrid.persistence.Schema.TYPE_ROLE;
+import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
 import org.apache.usergrid.persistence.SimpleEntityRef;
 import org.apache.usergrid.persistence.SimpleRoleRef;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_DICTIONARIES;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
+import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX_ENTRIES;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
+import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
 import org.apache.usergrid.persistence.cassandra.CassandraService;
+import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
 import org.apache.usergrid.persistence.cassandra.ConnectionRefImpl;
+import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchDeleteLocationInConnectionsIndex;
+import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchRemoveLocationFromCollectionIndex;
+import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchStoreLocationInCollectionIndex;
+import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchStoreLocationInConnectionsIndex;
 import org.apache.usergrid.persistence.cassandra.IndexUpdate;
+import static org.apache.usergrid.persistence.cassandra.IndexUpdate.indexValueCode;
+import static org.apache.usergrid.persistence.cassandra.IndexUpdate.toIndexableValue;
+import static org.apache.usergrid.persistence.cassandra.IndexUpdate.validIndexableValue;
 import org.apache.usergrid.persistence.cassandra.QueryProcessorImpl;
+import static org.apache.usergrid.persistence.cassandra.Serializers.be;
 import org.apache.usergrid.persistence.cassandra.index.ConnectedIndexScanner;
 import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
 import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
@@ -96,56 +130,18 @@ import org.apache.usergrid.persistence.query.ir.result.GeoIterator;
 import org.apache.usergrid.persistence.query.ir.result.SliceIterator;
 import org.apache.usergrid.persistence.query.ir.result.StaticIdIterator;
 import org.apache.usergrid.persistence.schema.CollectionInfo;
-import org.apache.usergrid.utils.IndexUtils;
-import org.apache.usergrid.utils.MapUtils;
-
-import com.yammer.metrics.annotation.Metered;
-
-import me.prettyprint.hector.api.Keyspace;
-import me.prettyprint.hector.api.beans.DynamicComposite;
-import me.prettyprint.hector.api.beans.HColumn;
-import me.prettyprint.hector.api.mutation.Mutator;
-import rx.Observable;
-
-import static java.util.Arrays.asList;
-
-import static me.prettyprint.hector.api.factory.HFactory.createMutator;
-import static org.apache.usergrid.persistence.Schema.COLLECTION_ROLES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_ENTITIES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTED_TYPES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_ENTITIES;
-import static org.apache.usergrid.persistence.Schema.DICTIONARY_CONNECTING_TYPES;
-import static org.apache.usergrid.persistence.Schema.INDEX_CONNECTIONS;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_CREATED;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_INACTIVITY;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
-import static org.apache.usergrid.persistence.Schema.PROPERTY_TITLE;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
-import static org.apache.usergrid.persistence.Schema.TYPE_ENTITY;
-import static org.apache.usergrid.persistence.Schema.TYPE_ROLE;
-import static org.apache.usergrid.persistence.Schema.getDefaultSchema;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_COMPOSITE_DICTIONARIES;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_DICTIONARIES;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX;
-import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_INDEX_ENTRIES;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addDeleteToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.addInsertToMutator;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.key;
-import static org.apache.usergrid.persistence.cassandra.CassandraService.INDEX_ENTRY_LIST_COUNT;
-import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchDeleteLocationInConnectionsIndex;
-import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchRemoveLocationFromCollectionIndex;
-import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchStoreLocationInCollectionIndex;
-import static org.apache.usergrid.persistence.cassandra.GeoIndexManager.batchStoreLocationInConnectionsIndex;
-import static org.apache.usergrid.persistence.cassandra.IndexUpdate.indexValueCode;
-import static org.apache.usergrid.persistence.cassandra.IndexUpdate.toIndexableValue;
-import static org.apache.usergrid.persistence.cassandra.IndexUpdate.validIndexableValue;
-import static org.apache.usergrid.persistence.cassandra.Serializers.be;
 import static org.apache.usergrid.utils.ClassUtils.cast;
 import static org.apache.usergrid.utils.CompositeUtils.setGreaterThanEqualityFlag;
+import org.apache.usergrid.utils.IndexUtils;
 import static org.apache.usergrid.utils.InflectionUtils.singularize;
+import org.apache.usergrid.utils.MapUtils;
 import static org.apache.usergrid.utils.MapUtils.addMapSet;
+import org.apache.usergrid.utils.UUIDUtils;
 import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.util.Assert;
+import rx.Observable;
 
 
 /**
@@ -155,11 +151,11 @@ public class CpRelationManager implements RelationManager {
 
     private static final Logger logger = LoggerFactory.getLogger( CpRelationManager.class );
 
-    public static final String ALL_TYPES = "zzzalltypeszzz";
+    static final String ALL_TYPES = "zzzalltypeszzz";
 
-    private static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
+    static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
 
-    private static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
+    static final String EDGE_CONN_SUFFIX = "zzzconnzzz";
 
     private CpEntityManagerFactory emf;
     
@@ -239,59 +235,6 @@ public class CpRelationManager implements RelationManager {
     }
 
     
-    static String getEdgeTypeFromConnectionType( String connectionType, String targetEntityType ) {
-
-        if ( connectionType != null && targetEntityType != null ) {
-            String csn = connectionType + "|" + targetEntityType + "|" + EDGE_CONN_SUFFIX;
-            return csn;
-        }
-
-        if ( connectionType != null ) {
-            // no suffix, this must be a search
-            String csn = connectionType;
-            return csn;
-        } 
-
-        return null;
-    }
-
-
-    static String getEdgeTypeFromCollectionName( String collectionName, String targetEntityType ) {
-
-        if ( collectionName != null && targetEntityType != null ) {
-            String csn = collectionName + "|" + targetEntityType + "|" + EDGE_COLL_SUFFIX;
-            return csn;
-        }
-
-        if ( collectionName != null ) {
-            // no suffix, this must be a search
-            String csn = collectionName;
-            return csn;
-        } 
-
-        return null;
-    }
-
-
-    static boolean isCollectionEdgeType( String type )  {
-        return type.endsWith( EDGE_COLL_SUFFIX );
-    }
-    
-    static boolean isConnectionEdgeType( String type )  {
-        return type.endsWith( EDGE_CONN_SUFFIX );
-    }
-    
-    public String getConnectionName( String edgeType ) {
-        String[] parts = edgeType.split("\\|");
-        return parts[0];
-    }
-
-    public String getCollectionName( String edgeType ) {
-        String[] parts = edgeType.split("\\|");
-        return parts[0];
-    }
-
-
     @Override
     public Set<String> getCollectionIndexes( String collectionName ) throws Exception {
         final Set<String> indexes = new HashSet<String>();
@@ -387,10 +330,10 @@ public class CpRelationManager implements RelationManager {
                     edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
 
                 String name = null;
-                if ( isConnectionEdgeType( edge.getType() )) {
-                    name = getConnectionName( edge.getType() );
+                if ( CpEntityManager.isConnectionEdgeType( edge.getType() )) {
+                    name = CpEntityManager.getConnectionType( edge.getType() );
                 } else {
-                    name = getCollectionName( edge.getType() );
+                    name = CpEntityManager.getCollectionName( edge.getType() );
                 }
                 addMapSet( results, eref, name );
             }
@@ -445,9 +388,9 @@ public class CpRelationManager implements RelationManager {
                 // reindex the entity in the source entity's collection or connection index
 
                 IndexScope indexScope;
-                if ( isCollectionEdgeType( edge.getType() )) {
+                if ( CpEntityManager.isCollectionEdgeType( edge.getType() )) {
 
-                    String collName = getCollectionName( edge.getType() ); 
+                    String collName = CpEntityManager.getCollectionName( edge.getType() ); 
                     indexScope = new IndexScopeImpl(
                         applicationScope.getApplication(),
                         new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
@@ -455,7 +398,7 @@ public class CpRelationManager implements RelationManager {
 
                 } else {
 
-                    String connName = getCollectionName( edge.getType() ); 
+                    String connName = CpEntityManager.getCollectionName( edge.getType() ); 
                     indexScope = new IndexScopeImpl(
                         applicationScope.getApplication(),
                         new SimpleId(sourceEntity.getUuid(), sourceEntity.getType()),
@@ -545,7 +488,7 @@ public class CpRelationManager implements RelationManager {
 
         Observable<Edge> edgesToTarget = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
             targetId,
-            CpRelationManager.getEdgeTypeFromConnectionType( connectionType, target.getType() ),
+            CpEntityManager.getEdgeTypeFromConnectionType( connectionType, target.getType() ),
             System.currentTimeMillis(), SearchByEdgeType.Order.DESCENDING,
             null)); // last
 
@@ -569,7 +512,7 @@ public class CpRelationManager implements RelationManager {
 
         Observable<Edge> edgesFromSource = gm.loadEdgesFromSource(new SimpleSearchByEdgeType(
             sourceId,
-            CpRelationManager.getEdgeTypeFromConnectionType( connectionType, null ),
+            CpEntityManager.getEdgeTypeFromConnectionType( connectionType, null ),
             System.currentTimeMillis(),SearchByEdgeType.Order.DESCENDING,
             null)); // last
 
@@ -587,12 +530,12 @@ public class CpRelationManager implements RelationManager {
         GraphManager gm = managerCache.getGraphManager(applicationScope);
 
         Observable<String> str = gm.getEdgeTypesFromSource( 
-                new SimpleSearchEdgeType( cpHeadEntity.getId(),null , null ));
+                new SimpleSearchEdgeType( cpHeadEntity.getId(), null , null ));
 
         Iterator<String> iter = str.toBlockingObservable().getIterator();
         while ( iter.hasNext() ) {
             String edgeType = iter.next();
-            indexes.add( getCollectionName( edgeType ) );
+            indexes.add( CpEntityManager.getCollectionName( edgeType ) );
         }
 
         return indexes;
@@ -670,7 +613,8 @@ public class CpRelationManager implements RelationManager {
         }
 
         if ( logger.isDebugEnabled() ) {
-            logger.debug("Loaded member entity {}:{} from scope\n   app {}\n   owner {}\n   name {} data {}", 
+            logger.debug("Loaded member entity {}:{} from scope\n   app {}\n   "
+                    + "owner {}\n   name {} data {}", 
                 new Object[] { 
                     itemRef.getType(), 
                     itemRef.getUuid(), 
@@ -683,12 +627,10 @@ public class CpRelationManager implements RelationManager {
 
         String edgeType = getEdgeTypeFromCollectionName( collName, memberEntity.getId().getType() );
 
-        logger.debug("addToCollection(): Creating edge type {} from {}:{} to {}:{}", 
-            new Object[] { 
-                edgeType, 
-                headEntity.getType(), headEntity.getUuid(), 
-                itemRef.getType(), itemRef.getUuid() });
-        UUID timeStampUuid =   memberEntity.getId().getUuid() != null &&  UUIDUtils.isTimeBased( memberEntity.getId().getUuid()) ?  memberEntity.getId().getUuid() : UUIDUtils.newTimeUUID();
+        UUID timeStampUuid =   memberEntity.getId().getUuid() != null 
+                &&  UUIDUtils.isTimeBased( memberEntity.getId().getUuid()) 
+                ?  memberEntity.getId().getUuid() : UUIDUtils.newTimeUUID();
+
         long uuidHash =    UUIDUtils.getUUIDLong(timeStampUuid);
 
         // create graph edge connection from head entity to member entity
@@ -700,37 +642,20 @@ public class CpRelationManager implements RelationManager {
         GraphManager gm = managerCache.getGraphManager(applicationScope);
         gm.writeEdge(edge).toBlockingObservable().last();
 
-        // index member into entity collection | type scope
-        IndexScope collectionIndexScope = new IndexScopeImpl(
-            applicationScope.getApplication(), 
-            cpHeadEntity.getId(), 
-            CpEntityManager.getCollectionScopeNameFromCollectionName( collName ));
-        EntityIndex collectionIndex = managerCache.getEntityIndex(collectionIndexScope);
-        collectionIndex.index( memberEntity );
-
-        // index member into entity | all-types scope
-        IndexScope entityAllTypesScope = new IndexScopeImpl(
-            applicationScope.getApplication(), 
-            cpHeadEntity.getId(), 
-            ALL_TYPES);
-        EntityIndex entityAllCollectionIndex = managerCache.getEntityIndex(entityAllTypesScope);
-        entityAllCollectionIndex.index( memberEntity );
+        logger.debug("\n\nWrote edgeType {}\n   from {}:{}\n   to {}:{}\n   scope {}:{}\n\n", new Object[] { 
+            edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(),
+            memberEntity.getId().getType(), memberEntity.getId().getUuid(),
+            applicationScope.getApplication().getType(), applicationScope.getApplication().getUuid()});  
 
-        // index member into application | all-types scope
-        IndexScope appAllTypesScope = new IndexScopeImpl(
-            applicationScope.getApplication(), 
-            applicationScope.getApplication(), 
-            ALL_TYPES);
-        EntityIndex allCollectionIndex = managerCache.getEntityIndex(appAllTypesScope);
-        allCollectionIndex.index( memberEntity );
+        ((CpEntityManager)em).indexEntityIntoCollection( cpHeadEntity, memberEntity, collName );
 
-        logger.debug("Added entity {}:{} to collection {}", new String[] { 
+        logger.debug("Added entity {}:{} to collection {}", new Object[] { 
             itemRef.getUuid().toString(), itemRef.getType(), collName }); 
 
-        logger.debug("With head entity scope is {}:{}:{}", new String[] { 
-            headEntityScope.getApplication().toString(), 
-            headEntityScope.getOwner().toString(),
-            headEntityScope.getName()}); 
+//        logger.debug("With head entity scope is {}:{}:{}", new Object[] { 
+//            headEntityScope.getApplication().toString(), 
+//            headEntityScope.getOwner().toString(),
+//            headEntityScope.getName()}); 
 
         if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
             getRelationManager( itemEntity )
@@ -969,53 +894,49 @@ public class CpRelationManager implements RelationManager {
         query.setEntityType( collection.getType() );
         query = adjustQuery( query );
 
-        CandidateResults crs = ei.search( query );
+        // Because of possible stale entities, which are filtered out by buildResults(), 
+        // we loop until the we've got enough results to satisfy the query limit. 
 
-        return buildResults( query, crs, collName );
-
-//        // Because of possible stale entities, which are filtered out by buildResults(), 
-//        // we loop until the we've got enough results to satisfy the query limit. 
-//
-//        int maxQueries = 10; // max re-queries to satisfy query limit
-//
-//        Results results = null;
-//        int queryCount = 0;
-//        int originalLimit = query.getLimit();
-//        boolean satisfied = false;
-//
-//        while ( !satisfied && queryCount++ < maxQueries ) {
-//
-//            CandidateResults crs = ei.search( query );
-//
-//            if ( results == null ) {
-//                results = buildResults( query, crs, collName );
-//
-//            } else {
-//                Results newResults = buildResults( query, crs, collName );
-//                results.merge( newResults );
-//            }
-//
-//            if ( crs.isEmpty() ) { // no more results
-//                satisfied = true;
-//
-//            } else if ( results.size() == query.getLimit() )  { // got what we need
-//                satisfied = true;
-//
-//            } else if ( crs.hasCursor() ) {
-//                satisfied = false;
-//
-//                // need to query for more
-//                // ask for just what we need to satisfy, don't want to exceed limit
-//                query.setCursor( results.getCursor() );
-//                query.setLimit( originalLimit - results.size() );
-//
-//                logger.warn("Satisfy query limit {}, new limit {} query count {}", new Object[] {
-//                    originalLimit, query.getLimit(), queryCount 
-//                });
-//            }
-//        }
-//
-//        return results;
+        int maxQueries = 10; // max re-queries to satisfy query limit
+
+        Results results = null;
+        int queryCount = 0;
+        int originalLimit = query.getLimit();
+        boolean satisfied = false;
+
+        while ( !satisfied && queryCount++ < maxQueries ) {
+
+            CandidateResults crs = ei.search( query );
+
+            if ( results == null ) {
+                results = buildResults( query, crs, collName );
+
+            } else {
+                Results newResults = buildResults( query, crs, collName );
+                results.merge( newResults );
+            }
+
+            if ( crs.isEmpty() ) { // no more results
+                satisfied = true;
+
+            } else if ( results.size() == query.getLimit() )  { // got what we need
+                satisfied = true;
+
+            } else if ( crs.hasCursor() ) {
+                satisfied = false;
+
+                // need to query for more
+                // ask for just what we need to satisfy, don't want to exceed limit
+                query.setCursor( results.getCursor() );
+                query.setLimit( originalLimit - results.size() );
+
+                logger.warn("Satisfy query limit {}, new limit {} query count {}", new Object[] {
+                    originalLimit, query.getLimit(), queryCount 
+                });
+            }
+        }
+
+        return results;
     }
 
 
@@ -1065,20 +986,9 @@ public class CpRelationManager implements RelationManager {
             new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() ))
                 .toBlockingObservable().last();
 
-        String edgeType = CpRelationManager
+        String edgeType = CpEntityManager
                 .getEdgeTypeFromConnectionType( connectionType, connectedEntityRef.getType() );
 
-        if ( logger.isDebugEnabled() ) {
-            logger.debug("createConnection(): "
-                    + "Creating edge type {} \n   from {}:{}\n   to {}:{}\n   in scope {}", 
-                new Object[] { 
-                    edgeType, 
-                    headEntity.getType(), headEntity.getUuid(), 
-                    connectedEntityRef.getType(), connectedEntityRef.getUuid(),
-                    applicationScope.getApplication()
-            });
-        }
-
         // create graph edge connection from head entity to member entity
         Edge edge = new SimpleEdge( 
             cpHeadEntity.getId(), 
@@ -1088,21 +998,13 @@ public class CpRelationManager implements RelationManager {
         GraphManager gm = managerCache.getGraphManager(applicationScope);
         gm.writeEdge(edge).toBlockingObservable().last();
 
-        // Index the new connection in app|source|type context
-        IndexScope indexScope = new IndexScopeImpl(
-            applicationScope.getApplication(), 
-            cpHeadEntity.getId(), 
-            CpEntityManager.getConnectionScopeName( connectedEntityRef.getType(), connectionType ));
-        EntityIndex ei = managerCache.getEntityIndex(indexScope);
-        ei.index( targetEntity );
+        logger.debug("\n\nWrote edgeType {}\n   from {}:{}\n   to {}:{}\n   scope {}:{}\n\n", new Object[] { 
+            edgeType, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid(),
+            targetEntity.getId().getType(), targetEntity.getId().getUuid(),
+            applicationScope.getApplication().getType(), applicationScope.getApplication().getUuid()}); 
 
-        // Index the new connection in app|scope|all-types context
-        IndexScope allTypesIndexScope = new IndexScopeImpl(
-            applicationScope.getApplication(), 
-            cpHeadEntity.getId(), 
-            ALL_TYPES);
-        EntityIndex aei = managerCache.getEntityIndex(allTypesIndexScope);
-        aei.index( targetEntity );
+        ((CpEntityManager)em).indexEntityIntoConnection( 
+            cpHeadEntity, targetEntity, connectedEntityRef.getType(), connectionType );
 
         Keyspace ko = cass.getApplicationKeyspace( applicationId );
         Mutator<ByteBuffer> m = createMutator( ko, be );
@@ -1338,7 +1240,9 @@ public class CpRelationManager implements RelationManager {
 
     @Override
     public Set<String> getConnectionTypes(boolean filterConnection) throws Exception {
-        Set<String> connections = cast( em.getDictionaryAsSet( headEntity, Schema.DICTIONARY_CONNECTED_TYPES ) );
+        Set<String> connections = cast( 
+                em.getDictionaryAsSet( headEntity, Schema.DICTIONARY_CONNECTED_TYPES ) );
+
         if ( connections == null ) {
             return null;
         }
@@ -1435,7 +1339,7 @@ public class CpRelationManager implements RelationManager {
 
         // looking for edges to the head entity
         String edgeType = 
-                CpRelationManager.getEdgeTypeFromConnectionType( connType, headEntity.getType() );
+                CpEntityManager.getEdgeTypeFromConnectionType( connType, headEntity.getType() );
 
         Map<EntityRef, Set<String>> containers = 
             getContainers( count, edgeType, fromEntityType );
@@ -2522,4 +2426,6 @@ public class CpRelationManager implements RelationManager {
     }
 
 
+
+
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index 2a5da5e..9063775 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -17,32 +17,21 @@
 package org.apache.usergrid.corepersistence;
 
 
-import java.io.IOException;
-import java.util.Properties;
-import java.util.UUID;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.usergrid.mq.cassandra.QueuesCF;
-import org.apache.usergrid.persistence.EntityManagerFactory;
-import org.apache.usergrid.persistence.cassandra.ApplicationCF;
-import org.apache.usergrid.persistence.cassandra.CassandraService;
-import org.apache.usergrid.persistence.cassandra.Setup;
-import org.apache.usergrid.persistence.core.migration.MigrationException;
-import org.apache.usergrid.persistence.core.migration.MigrationManager;
-import org.apache.usergrid.persistence.entities.Application;
-
 import com.google.inject.Guice;
 import com.google.inject.Injector;
 import com.netflix.config.ConfigurationManager;
-
+import java.util.Properties;
+import java.util.UUID;
+import java.util.logging.Level;
 import me.prettyprint.cassandra.service.CassandraHost;
 import me.prettyprint.hector.api.ddl.ComparatorType;
-
 import static me.prettyprint.hector.api.factory.HFactory.createColumnFamilyDefinition;
 import org.apache.commons.lang.StringUtils;
+import org.apache.usergrid.mq.cassandra.QueuesCF;
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.cassandra.ApplicationCF;
 import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.getCfDefs;
+import org.apache.usergrid.persistence.cassandra.CassandraService;
 import static org.apache.usergrid.persistence.cassandra.CassandraService.APPLICATIONS_CF;
 import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_APPLICATION;
 import static org.apache.usergrid.persistence.cassandra.CassandraService.DEFAULT_ORGANIZATION;
@@ -54,6 +43,13 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.SYSTEM_
 import static org.apache.usergrid.persistence.cassandra.CassandraService.TOKENS_CF;
 import static org.apache.usergrid.persistence.cassandra.CassandraService.USE_VIRTUAL_KEYSPACES;
 import static org.apache.usergrid.persistence.cassandra.CassandraService.keyspaceForApplication;
+import org.apache.usergrid.persistence.cassandra.Setup;
+import org.apache.usergrid.persistence.core.migration.MigrationException;
+import org.apache.usergrid.persistence.core.migration.MigrationManager;
+import org.apache.usergrid.persistence.entities.Application;
+import org.apache.usergrid.persistence.exceptions.ApplicationAlreadyExistsException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 
 /**
@@ -168,11 +164,19 @@ public class CpSetup implements Setup {
 
         logger.info("Setting up default applications");
 
-        emf.initializeApplication( DEFAULT_ORGANIZATION, 
-                emf.getDefaultAppId(), DEFAULT_APPLICATION, null );
+        try {
+            emf.initializeApplication( DEFAULT_ORGANIZATION,
+                    emf.getDefaultAppId(), DEFAULT_APPLICATION, null );
+        } catch (ApplicationAlreadyExistsException ex) {
+            logger.warn("Application {}/{} already exists", DEFAULT_ORGANIZATION, DEFAULT_APPLICATION);
+        }
 
-        emf.initializeApplication( DEFAULT_ORGANIZATION, 
-                emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
+        try {
+            emf.initializeApplication( DEFAULT_ORGANIZATION,
+                    emf.getManagementAppId(), MANAGEMENT_APPLICATION, null );
+        } catch (ApplicationAlreadyExistsException ex) {
+            logger.warn("Application {}/{} already exists", DEFAULT_ORGANIZATION, MANAGEMENT_APPLICATION);
+        }
     }
 
 
@@ -181,12 +185,7 @@ public class CpSetup implements Setup {
         return SystemDefaults.managementApp;
     }
 
-
-    /** @return statically constructed reference to the default application */
-//    public static Application getDefaultApp() {
-//        return SystemDefaults.defaultApp;
-//    }
-
+    
     @Override
     public void setupSystemKeyspace() throws Exception {
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
index 8002199..5897d6c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/HybridEntityManagerFactory.java
@@ -150,8 +150,17 @@ public class HybridEntityManagerFactory implements EntityManagerFactory, Applica
     }
 
     @Override
-    public void rebuildCollectionIndex(UUID appId, String collectionName, ProgressObserver po) 
-            throws Exception {
-        factory.rebuildCollectionIndex(appId, collectionName, po);
+    public void rebuildAllIndexes(ProgressObserver po) throws Exception {
+        factory.rebuildAllIndexes(po);
+    }
+
+    @Override
+    public void rebuildApplicationIndexes(UUID appId, ProgressObserver po) throws Exception {
+        factory.rebuildApplicationIndexes(appId, po);
+    }
+
+    @Override
+    public void rebuildCollectionIndex(UUID appId, String collection, Object object) {
+        factory.rebuildCollectionIndex(appId, collection, object);
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index bf4e47e..d962d2a 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -117,17 +117,20 @@ public interface EntityManagerFactory {
 
     public void refreshIndex();
 
+    public void rebuildAllIndexes( ProgressObserver po ) throws Exception;
+    
     public void rebuildInternalIndexes( ProgressObserver po ) throws Exception;
 
-    public void rebuildCollectionIndex( 
-        UUID appId, String collectionName, ProgressObserver po ) throws Exception;
+    public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception;
 
     public void setApplicationContext(ApplicationContext ac);
 
     /** For testing purposes */
     public void flushEntityManagerCaches();
 
+    public void rebuildCollectionIndex(UUID appId, String collection, Object object);
+
     public interface ProgressObserver {
-        public void onProgress();
+        public void onProgress( EntityRef source, EntityRef target, String edgeType );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
index 987d39a..1e3b1c1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
@@ -28,9 +28,6 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 import java.util.UUID;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -67,7 +64,6 @@ import me.prettyprint.hector.api.ddl.KeyspaceDefinition;
 import me.prettyprint.hector.api.factory.HFactory;
 import me.prettyprint.hector.api.mutation.Mutator;
 import me.prettyprint.hector.api.query.ColumnQuery;
-import me.prettyprint.hector.api.query.CountQuery;
 import me.prettyprint.hector.api.query.MultigetSliceQuery;
 import me.prettyprint.hector.api.query.QueryResult;
 import me.prettyprint.hector.api.query.RangeSlicesQuery;
@@ -84,7 +80,6 @@ import static org.apache.commons.collections.MapUtils.getIntValue;
 import static org.apache.commons.collections.MapUtils.getString;
 import static org.apache.usergrid.persistence.cassandra.ApplicationCF.ENTITY_ID_SETS;
 import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.batchExecute;
-import static org.apache.usergrid.persistence.cassandra.CassandraPersistenceUtils.buildSetIdListMutator;
 import static org.apache.usergrid.utils.ConversionUtils.bytebuffer;
 import static org.apache.usergrid.utils.ConversionUtils.bytebuffers;
 import static org.apache.usergrid.utils.JsonUtils.mapToFormattedJsonString;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index 89d8a07..955e707 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -56,7 +56,6 @@ import static java.lang.String.CASE_INSENSITIVE_ORDER;
 
 
 import static me.prettyprint.hector.api.factory.HFactory.createRangeSlicesQuery;
-import org.apache.usergrid.persistence.Results;
 import static org.apache.usergrid.persistence.Schema.PROPERTY_NAME;
 import static org.apache.usergrid.persistence.Schema.PROPERTY_UUID;
 import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
@@ -68,8 +67,6 @@ import static org.apache.usergrid.persistence.cassandra.CassandraService.PROPERT
 import static org.apache.usergrid.persistence.cassandra.CassandraService.RETRY_COUNT;
 import static org.apache.usergrid.utils.ConversionUtils.uuid;
 import static org.apache.usergrid.persistence.cassandra.Serializers.*;
-import org.apache.usergrid.persistence.exceptions.DuplicateUniquePropertyExistsException;
-import org.apache.usergrid.persistence.index.query.Query;
 
 
 /**
@@ -423,49 +420,22 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
     }
 
     @Override
-    public void rebuildCollectionIndex(UUID appId, String collectionName, ProgressObserver po ) 
-            throws Exception {
-
-        logger.info( "Reindexing collection: {} for app id: {}", collectionName, appId );
-
-        EntityManager em = getEntityManager( appId );
-        Application app = em.getApplication();
-
-        // search for all orgs
-
-        Query query = new Query();
-        query.setLimit(REBUILD_PAGE_SIZE );
-        Results r = null;
-
-        do {
-
-            r = em.searchCollection( app, collectionName, query );
-
-            for ( org.apache.usergrid.persistence.Entity entity : r.getEntities() ) {
-                logger.info( "Updating entity type: {} with id: {} for app id: {}", new Object[] {
-                        entity.getType(), entity.getUuid(), appId
-                } );
-
-                try {
-                    em.update( entity );
-                }
-                catch ( DuplicateUniquePropertyExistsException dupee ) {
-                    logger.error( "duplicate property for type: {} with id: {} for app id: {}.  "
-                            + "Property name: {} , value: {}", new Object[] {
-                            entity.getType(), entity.getUuid(), appId, dupee.getPropertyName(), 
-                            dupee.getPropertyValue()
-                    } );
-                }
-            }
+    public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
+        throw new UnsupportedOperationException("Not supported."); 
+    }
 
-            query.setCursor( r.getCursor() );
-        }
-        while ( r != null && r.size() == REBUILD_PAGE_SIZE );
+    @Override
+    public void rebuildAllIndexes(ProgressObserver po) throws Exception {
+        throw new UnsupportedOperationException("Not supported."); 
+    }
 
+    @Override
+    public void rebuildApplicationIndexes(UUID appId, ProgressObserver po) throws Exception {
+        throw new UnsupportedOperationException("Not supported."); 
     }
 
     @Override
-    public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
-        // no op
+    public void rebuildCollectionIndex(UUID appId, String collection, Object object) {
+        throw new UnsupportedOperationException("Not supported."); 
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 35d1d50..7af66d3 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -17,9 +17,6 @@
 package org.apache.usergrid.persistence;
 
 
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import org.junit.After;
@@ -37,7 +34,11 @@ import com.codahale.metrics.Meter;
 import com.codahale.metrics.MetricRegistry;
 import com.codahale.metrics.Slf4jReporter;
 import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import org.apache.usergrid.cassandra.Concurrent;
+import static org.junit.Assert.fail;
 
 
 //@RunWith(JukitoRunner.class)
@@ -47,21 +48,21 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
     private static final Logger logger = LoggerFactory.getLogger(PerformanceEntityRebuildIndexTest.class );
 
     private static final MetricRegistry registry = new MetricRegistry();
+    private Slf4jReporter reporter;
 
     private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
 
-    private static final long writeDelayMs = 7;
-    private static final long readDelayMs = 7;
+    private static final long writeDelayMs = 9;
+    //private static final long readDelayMs = 7;
 
     @Rule
     public Application app = new CoreApplication( setup );
 
-    private Slf4jReporter reporter;
-
 
     @Before
     public void startReporting() {
 
+        logger.debug("Starting metrics reporting");
         reporter = Slf4jReporter.forRegistry( registry ).outputTo( logger )
                 .convertRatesTo( TimeUnit.SECONDS )
                 .convertDurationsTo( TimeUnit.MILLISECONDS ).build();
@@ -72,13 +73,15 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
 
     @After
     public void printReport() {
+
+        logger.debug("Printing metrics report");
         reporter.report();
         reporter.stop();
     }
 
 
     @Test
-    public void rebuildIndex() throws Exception {
+    public void rebuildIndex() {
 
         logger.info("Started rebuildIndex()");
 
@@ -96,34 +99,55 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
         while ( System.currentTimeMillis() < stopTime ) {
 
             entityMap.put( "key", i );
-            final Entity created = em.create("testType", entityMap );
+            final Entity created;
+            try {
+                created = em.create("testType", entityMap );
+            } catch (Exception ex) {
+                throw new RuntimeException("Error creating entity", ex);
+            }
 
             entityRefs.add( new SimpleEntityRef( created.getType(), created.getUuid() ) );
 
+            if ( i % 100 == 0 ) {
+                logger.info("Created {} entities", i );
+            }
             i++;
 
-            if ( i % 1000 == 0 ) {
-                logger.debug("rebuildIndex() Created {} entities",i );
-            }
-            Thread.sleep( writeDelayMs );
+            try { Thread.sleep( writeDelayMs ); } catch (InterruptedException ignored ) {}
         }
-        logger.info("rebuildIndex() Created {} entities", i);
+        logger.info("Created {} entities", i);
+
 
         final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
         final Meter meter = registry.meter( meterName );
-
+        
         EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+            int counter = 0;
             @Override
-            public void onProgress() {
+            public void onProgress( EntityRef s, EntityRef t, String etype ) {
+
                 meter.mark();
+
+                logger.debug("Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
+                    s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
+
+                if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
+                    logger.info("Reindexed {} entities", counter );
+                }
+                counter++;
             }
         };
 
-        setup.getEmf().rebuildInternalIndexes( po );
+        try {
+            setup.getEmf().rebuildAllIndexes( po );
 
-        setup.getEmf().rebuildCollectionIndex( app.getId(), "testTypes", po);
+            registry.remove( meterName );
+            logger.info("Finished rebuildIndex()");
+
+        } catch (Exception ex) {
+            logger.error("Error rebuilding index", ex);
+            fail();
+        }
 
-        registry.remove( meterName );
-        logger.info("Finished rebuildIndex()");
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1aa04a71/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
----------------------------------------------------------------------
diff --git a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
index 81d0c1c..66a5cfb 100644
--- a/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
+++ b/stack/tools/src/main/java/org/apache/usergrid/tools/IndexRebuild.java
@@ -33,12 +33,13 @@ import org.apache.commons.cli.CommandLine;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.OptionBuilder;
 import org.apache.commons.cli.Options;
-
+import org.apache.usergrid.persistence.EntityManagerFactory;
+import org.apache.usergrid.persistence.EntityRef;
 
 
 /**
- * This is a utility to load all entities in an application and re-save them, this forces 
- * the secondary indexing to be updated.
+ * Index rebuild utility for Usergrid. Can be used to rebuild the index for a specific 
+ * application, a specific application's collection or for an entire Usergrid system.
  */
 public class IndexRebuild extends ToolBase {
 
@@ -46,6 +47,8 @@ public class IndexRebuild extends ToolBase {
 
     private static final String COLLECTION_ARG = "col";
 
+    private static final String ALL_ARG = "all";
+
     private static final int PAGE_SIZE = 100;
 
 
@@ -71,12 +74,16 @@ public class IndexRebuild extends ToolBase {
         Option collOpt = OptionBuilder.withArgName( COLLECTION_ARG ).hasArg().isRequired( false )
                 .withDescription( "Collection name" ).create( COLLECTION_ARG );
 
+        Option allOpt = OptionBuilder.withType( Boolean.class ).withArgName( ALL_ARG ).hasArg().isRequired( false )
+                .withDescription( "True to reindex all application" ).create( ALL_ARG );
+
         Options options = new Options();
         options.addOption( hostOpt );
         options.addOption( esHostsOpt );
         options.addOption( esClusterOpt );
         options.addOption( appOpt );
         options.addOption( collOpt );
+        options.addOption( allOpt );
 
         return options;
     }
@@ -94,20 +101,40 @@ public class IndexRebuild extends ToolBase {
 
         logger.info( "Starting index rebuild" );
 
-        emf.rebuildInternalIndexes( null );
+        EntityManagerFactory.ProgressObserver po = new EntityManagerFactory.ProgressObserver() {
+            @Override
+            public void onProgress(EntityRef s, EntityRef t, String etype) {
+                logger.info("Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
+                    s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
+            }
+        };
+
+        emf.rebuildInternalIndexes( po ); 
         emf.refreshIndex();
 
-        /**
-         * Goes through each app id specified
-         */
-        for ( UUID appId : getAppIds( line ) ) {
+        if ( line.getOptionValue("all") != null && line.getOptionValue("all").equalsIgnoreCase("true") ) {
+            emf.rebuildAllIndexes( po );
+
+        } else if ( line.getOptionValue( APPLICATION_ARG ) != null ) {
+
+            // Goes through each app id specified
+            for ( UUID appId : getAppIds( line ) ) {
+
+                logger.info( "Reindexing for app id: {}", appId );
+                Set<String> collections = getCollections( line, appId );
 
-            logger.info( "Reindexing for app id: {}", appId );
-            Set<String> collections = getCollections( line, appId );
+                for ( String collection : collections ) {
+                    emf.rebuildCollectionIndex( appId, collection, po );
+                    emf.refreshIndex();
+                }
+            }
+
+        } else {
 
-            for ( String collection : collections ) {
-                emf.rebuildCollectionIndex( appId, collection, null );
-                emf.refreshIndex();
+            Map<String, UUID> ids = emf.getApplications();
+            System.out.println( "Printing all apps" );
+            for ( Entry<String, UUID> entry : ids.entrySet() ) {
+                System.out.println( entry.getKey() + " appid=" + entry.getValue() );
             }
         }
 
@@ -117,27 +144,20 @@ public class IndexRebuild extends ToolBase {
 
     /** Get all app id */
     private Collection<UUID> getAppIds( CommandLine line ) throws Exception {
+
         String appId = line.getOptionValue( APPLICATION_ARG );
 
         Map<String, UUID> ids = emf.getApplications();
 
         if ( appId != null ) {
-
             UUID id = UUIDUtils.tryExtractUUID( appId );
-
             if ( id == null ) {
                 logger.debug("Got applications: " + ids );
                 id = emf.getApplications().get( appId );
             }
-
             return Collections.singleton( id );
         }
 
-        System.out.println( "Printing all apps" );
-        for ( Entry<String, UUID> entry : ids.entrySet() ) {
-            System.out.println( entry.getKey() + " appid=" + entry.getValue() );
-        }
-
         return ids.values();
     }