You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/11/12 20:11:42 UTC

[01/27] incubator-usergrid git commit: Changed interface for types and updated internals

Repository: incubator-usergrid
Updated Branches:
  refs/heads/two-dot-o 84b836515 -> d2d1fe74f


Changed interface for types and updated internals


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2db132c3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2db132c3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2db132c3

Branch: refs/heads/two-dot-o
Commit: 2db132c323a1165234d0556d7d252bf244e34f17
Parents: 795874e
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 5 22:09:50 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 6 17:43:15 2014 -0700

----------------------------------------------------------------------
 .../CpEntityIndexDeleteListener.java            |   2 +-
 .../corepersistence/CpEntityManager.java        |  14 +-
 .../corepersistence/CpRelationManager.java      |  26 +-
 .../results/FilteringLoader.java                |   2 +-
 .../corepersistence/StaleIndexCleanupTest.java  |   2 +-
 .../usergrid/persistence/index/EntityIndex.java |   4 +-
 .../usergrid/persistence/index/SearchTypes.java |  59 ++
 .../index/impl/EsEntityIndexBatchImpl.java      | 101 +--
 .../index/impl/EsEntityIndexImpl.java           |  40 +-
 .../persistence/index/impl/IndexScopeImpl.java  |  14 +-
 .../persistence/index/impl/IndexingUtils.java   |  35 +-
 .../index/impl/CorePerformanceIT.java           |  36 +-
 .../impl/EntityConnectionIndexImplTest.java     | 121 ++-
 .../persistence/index/impl/EntityIndexTest.java | 775 +++++++++----------
 14 files changed, 709 insertions(+), 522 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
index 125b90b..13becc4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
@@ -65,7 +65,7 @@ public class CpEntityIndexDeleteListener {
 
         final CollectionScope collectionScope = event.getCollectionScope();
         final IndexScope indexScope = 
-                new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName());
+                new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName(), entityType );
         final EntityIndex entityIndex = entityIndexFactory.createEntityIndex(
                 new ApplicationScopeImpl( collectionScope.getApplication()));
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index daa046c..97bdba8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -637,7 +637,7 @@ public class CpEntityManager implements EntityManager {
 
                         IndexScope indexScope = new IndexScopeImpl(
                                 new SimpleId( uuid, ownerType ), 
-                                CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ) );
+                                CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ), entityType );
 
 
                         batch.index( indexScope, entity );
@@ -650,13 +650,13 @@ public class CpEntityManager implements EntityManager {
             // deindex from default index scope
             IndexScope defaultIndexScope = new IndexScopeImpl(
                     getApplicationScope().getApplication(),
-                    CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+                    CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ), entityType );
 
             batch.deindex(defaultIndexScope,  entity );
 
             IndexScope allTypesIndexScope = new IndexScopeImpl(
                 getApplicationScope().getApplication(), 
-                    CpNamingUtils.ALL_TYPES);
+                    CpNamingUtils.ALL_TYPES, entityType );
 
             batch.deindex( allTypesIndexScope,  entity );
 
@@ -1048,7 +1048,7 @@ public class CpEntityManager implements EntityManager {
 
         IndexScope defaultIndexScope = new IndexScopeImpl(
                 getApplicationScope().getApplication(), 
-                CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+                CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ), entityType );
 
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
         EntityIndex ei = managerCache.getEntityIndex(getApplicationScope());
@@ -2976,21 +2976,21 @@ public class CpEntityManager implements EntityManager {
         // index member into entity collection | type scope
         IndexScope collectionIndexScope = new IndexScopeImpl(
                 collectionEntity.getId(),
-                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
 
         batch.index(collectionIndexScope, memberEntity);
         
         // index member into entity | all-types scope
         IndexScope entityAllTypesScope = new IndexScopeImpl(
                 collectionEntity.getId(),
-                CpNamingUtils.ALL_TYPES);
+                CpNamingUtils.ALL_TYPES, entityType );
 
         batch.index(entityAllTypesScope, memberEntity);
         
         // index member into application | all-types scope
         IndexScope appAllTypesScope = new IndexScopeImpl(
                 getApplicationScope().getApplication(),
-                CpNamingUtils.ALL_TYPES);
+                CpNamingUtils.ALL_TYPES, entityType );
 
         batch.index(appAllTypesScope, memberEntity);
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 1e8dcc3..c33c032 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -413,7 +413,7 @@ public class CpRelationManager implements RelationManager {
                             String collName = CpNamingUtils.getCollectionName( edge.getType() );
                             indexScope = new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(),
                                 sourceEntity.getType() ),
-                                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+                                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
                         }
                         else {
 
@@ -421,7 +421,7 @@ public class CpRelationManager implements RelationManager {
                             indexScope = new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(),
                                 sourceEntity.getType() ),
                                 CpNamingUtils.getConnectionScopeName( cpEntity.getId().getType(),
-                                connName ) );
+                                connName ), entityType );
                         }
 
                         entityIndexBatch.index( indexScope, cpEntity );
@@ -429,7 +429,7 @@ public class CpRelationManager implements RelationManager {
                         // reindex the entity in the source entity's all-types index
 
                         indexScope = new IndexScopeImpl( new SimpleId( 
-                            sourceEntity.getUuid(), sourceEntity.getType() ), CpNamingUtils.ALL_TYPES );
+                            sourceEntity.getUuid(), sourceEntity.getType() ), CpNamingUtils.ALL_TYPES, entityType );
 
                         entityIndexBatch.index( indexScope, cpEntity );
                     }
@@ -806,7 +806,7 @@ public class CpRelationManager implements RelationManager {
         // remove item from collection index
         IndexScope indexScope = new IndexScopeImpl(
             cpHeadEntity.getId(), 
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
 
         batch.deindex( indexScope, memberEntity );
 
@@ -814,7 +814,7 @@ public class CpRelationManager implements RelationManager {
         IndexScope itemScope = new IndexScopeImpl(
             memberEntity.getId(), 
             CpNamingUtils.getCollectionScopeNameFromCollectionName(
-                    Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ));
+                    Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ), entityType );
 
 
         batch.deindex( itemScope, cpHeadEntity );
@@ -915,7 +915,7 @@ public class CpRelationManager implements RelationManager {
 
         IndexScope indexScope = new IndexScopeImpl(
             cpHeadEntity.getId(), 
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
 
         EntityIndex ei = managerCache.getEntityIndex( applicationScope );
 
@@ -1034,12 +1034,12 @@ public class CpRelationManager implements RelationManager {
 
         // Index the new connection in app|source|type context
         IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(),
-                CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ) );
+                CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ), entityType );
 
         batch.index( indexScope, targetEntity );
 
         // Index the new connection in app|scope|all-types context
-        IndexScope allTypesIndexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.ALL_TYPES );
+        IndexScope allTypesIndexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.ALL_TYPES, entityType );
         batch.index( allTypesIndexScope, targetEntity );
 
 
@@ -1267,14 +1267,14 @@ public class CpRelationManager implements RelationManager {
             new SimpleId( connectingEntityRef.getUuid(),
                 connectingEntityRef.getType() ),
                 CpNamingUtils.getConnectionScopeName( targetEntity.getId().getType(),
-                connectionType ) );
+                connectionType ), entityType );
         batch.deindex( indexScope, targetEntity );
 
         // Deindex the connection in app|source|type context
         IndexScope allTypesIndexScope = new IndexScopeImpl( 
             new SimpleId( connectingEntityRef.getUuid(),
                 connectingEntityRef.getType() ),
-                CpNamingUtils.ALL_TYPES );
+                CpNamingUtils.ALL_TYPES, entityType );
 
         batch.deindex( allTypesIndexScope, targetEntity );
 
@@ -1334,7 +1334,7 @@ public class CpRelationManager implements RelationManager {
                 scopeName = CpNamingUtils.ALL_TYPES;
             }
 
-            IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), scopeName );
+            IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), scopeName, entityType );
 
             final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
 
@@ -1423,7 +1423,7 @@ public class CpRelationManager implements RelationManager {
             // search across all types of collections of the head-entity
             IndexScope indexScope = new IndexScopeImpl( 
                 cpHeadEntity.getId(), 
-                CpNamingUtils.ALL_TYPES );
+                CpNamingUtils.ALL_TYPES, entityType );
 
             EntityIndex ei = managerCache.getEntityIndex( applicationScope );
 
@@ -1440,7 +1440,7 @@ public class CpRelationManager implements RelationManager {
         IndexScope indexScope = new IndexScopeImpl( 
             cpHeadEntity.getId(), 
             CpNamingUtils.getConnectionScopeName( query.getEntityType(), 
-            query.getConnectionType() ) );
+            query.getConnectionType() ), entityType );
         EntityIndex ei = managerCache.getEntityIndex( applicationScope );
         
         logger.debug("Searching connections from the scope {}:{}",

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 8ca2211..c35cb33 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -241,7 +241,7 @@ public class FilteringLoader implements ResultsLoader {
 
         IndexScope indexScope = new IndexScopeImpl( 
             ownerId,
-            CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()));
+            CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()), entityType );
 
         batch.deindex( indexScope, candidateResult );
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 906d7b8..4cd77e5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -240,7 +240,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
 
         IndexScope is = new IndexScopeImpl(
             new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
         Query rcq = Query.fromQL(query);
         rcq.setLimit(10000); // no paging
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 8ded788..1a6adb6 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -44,7 +44,7 @@ public interface EntityIndex {
     /**
      * Execute query in Usergrid syntax.
      */
-    public CandidateResults search(final IndexScope indexScope,  Query query );
+    public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query );
 
     /**
      * Get the candidate results of all versions of the entity for this id.
@@ -67,3 +67,5 @@ public interface EntityIndex {
     public Health getIndexHealth();
 
 }
+
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
new file mode 100644
index 0000000..584d65b
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -0,0 +1,59 @@
+package org.apache.usergrid.persistence.index;/*
+ *
+ *  * Licensed to the Apache Software Foundation (ASF) under one
+ *  * or more contributor license agreements.  See the NOTICE file
+ *  * distributed with this work for additional information
+ *  * regarding copyright ownership.  The ASF licenses this file
+ *  * to you under the Apache License, Version 2.0 (the
+ *  * "License"); you may not use this file except in compliance
+ *  * with the License.  You may obtain a copy of the License at
+ *  *
+ *  *    http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing,
+ *  * software distributed under the License is distributed on an
+ *  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ *  * KIND, either express or implied.  See the License for the
+ *  * specific language governing permissions and limitations
+ *  * under the License.
+ *
+ */
+
+
+/**
+ * Class to encapsulate search types
+ */
+
+public class SearchTypes {
+
+    private static final SearchTypes ALL_TYPES = new SearchTypes(  );
+
+    private final String[] types;
+
+
+    private SearchTypes( final String... types ) {this.types = types;}
+
+
+    public String[] getTypeNames() {
+        return types;
+    }
+
+
+    /**
+     * Create a search that will search on the specified types
+     * @param types
+     * @return
+     */
+    public static SearchTypes fromTypes( final String... types ) {
+        return new SearchTypes( types );
+    }
+
+
+    /**
+     * Get a search that will search all types in the specified context
+     * @return
+     */
+    public static SearchTypes allTypes(){
+        return ALL_TYPES;
+    }
+}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index ad58fa8..5a19d1e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -62,12 +62,13 @@ import com.google.common.base.Joiner;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
 
 
 public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -110,36 +111,33 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         IndexValidationUtils.validateIndexScope( indexScope );
 
-        final String indexType = createCollectionScopeTypeName( indexScope );
+        final String context = createContextName( indexScope );
+        final String entityType = entity.getId().getType();
 
         if ( log.isDebugEnabled() ) {
             log.debug( "Indexing entity {}:{} in scope\n   app {}\n   "
-                + "owner {}\n   name {}\n   type {}", new Object[] {
-                    entity.getId().getType(), 
-                    entity.getId().getUuid(), 
-                    applicationScope.getApplication(), 
-                    indexScope.getOwner(), 
-                    indexScope.getName(), indexType
-            });
+                    + "owner {}\n   name {}\n   type {} \n scope type{}", new Object[] {
+                    entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
+                    indexScope.getOwner(), indexScope.getName(), entityType, context
+            } );
         }
 
         ValidationUtils.verifyEntityWrite( entity );
 
-        Map<String, Object> entityAsMap = entityToMap( entity );
+        Map<String, Object> entityAsMap = entityToMap( entity, context );
+
+        // need prefix here because we index UUIDs as strings
 
-        // need prefix here becuase we index UUIDs as strings
-        entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME, 
-                entity.getId().getUuid().toString().toLowerCase() );
 
         // let caller add these fields if needed
         // entityAsMap.put("created", entity.getId().getUuid().timestamp();
         // entityAsMap.put("updated", entity.getVersion().timestamp());
 
-        String indexId = createIndexDocId( entity );
+        String indexId = createIndexDocId( entity, context );
 
         log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
 
-        bulkRequest.add(client.prepareIndex( indexName, indexType, indexId).setSource(entityAsMap));
+        bulkRequest.add( client.prepareIndex( indexName, entityType, indexId ).setSource( entityAsMap ) );
 
         maybeFlush();
 
@@ -152,23 +150,21 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         IndexValidationUtils.validateIndexScope( indexScope );
 
-        final String indexType = createCollectionScopeTypeName( indexScope );
+        final String context = createContextName( indexScope );
+        final String entityType = id.getType();
+
 
         if ( log.isDebugEnabled() ) {
-            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   "
-                + "name {} type {}", new Object[] {
-                    id.getType(), 
-                    id.getUuid(), 
-                    applicationScope.getApplication(), 
-                    indexScope.getOwner(),
-                    indexScope.getName(), 
-                    indexType
-            } );
+            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   " + "name {} context{}, type {}",
+                    new Object[] {
+                            id.getType(), id.getUuid(), applicationScope.getApplication(), indexScope.getOwner(),
+                            indexScope.getName(), context, entityType
+                    } );
         }
 
-        String indexId = createIndexDocId( id, version );
+        String indexId = createIndexDocId( id, version, context );
 
-        bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh(refresh));
+        bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
 
         log.debug( "Deindexed Entity with index id " + indexId );
 
@@ -204,7 +200,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
     private void execute( final BulkRequestBuilder request ) {
 
         //nothing to do, we haven't added anthing to the index
-        if(request.numberOfActions() == 0){
+        if ( request.numberOfActions() == 0 ) {
             return;
         }
 
@@ -212,7 +208,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         try {
             responses = request.execute().actionGet();
-        }catch(Throwable t){
+        }
+        catch ( Throwable t ) {
             log.error( "Unable to communicate with elasticsearch" );
             failureMonitor.fail( "Unable to execute batch", t );
             throw t;
@@ -223,8 +220,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         for ( BulkItemResponse response : responses ) {
             if ( response.isFailed() ) {
-                throw new RuntimeException("Unable to index documents.  Errors are :" 
-                        + response.getFailure().getMessage() );
+                throw new RuntimeException(
+                        "Unable to index documents.  Errors are :" + response.getFailure().getMessage() );
             }
         }
 
@@ -249,11 +246,30 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
 
     /**
+     * Set the entity as a map with the context
+     * @param entity The entity
+     * @param context The context this entity appears in
+     * @return
+     */
+    private static Map entityToMap( final Entity entity, final String context ) {
+        final Map entityMap = entityToMap( entity );
+
+        //add the context for filtering later
+        entityMap.put( ENTITY_CONTEXT, context );
+
+        //but the fieldname
+        entityMap.put( ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() );
+
+        return entityMap;
+    }
+
+
+    /**
      * Convert Entity to Map and Adding prefixes for types:
      * <pre>
-     * su_ - String unanalyzed field 
-     * sa_ - String analyzed field 
-     * go_ - Location field nu_ - Number field 
+     * su_ - String unanalyzed field
+     * sa_ - String analyzed field
+     * go_ - Location field nu_ - Number field
      * bu_ - Boolean field
      * </pre>
      */
@@ -267,8 +283,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
             if ( f instanceof ListField ) {
                 List list = ( List ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), 
-                        new ArrayList( processCollectionForMap( list ) ) );
+                entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
 
                 if ( !list.isEmpty() ) {
                     if ( list.get( 0 ) instanceof String ) {
@@ -281,13 +296,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
             }
             else if ( f instanceof ArrayField ) {
                 List list = ( List ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), 
-                        new ArrayList( processCollectionForMap( list ) ) );
+                entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
             }
             else if ( f instanceof SetField ) {
                 Set set = ( Set ) field.getValue();
-                entityMap.put( field.getName().toLowerCase(), 
-                        new ArrayList( processCollectionForMap( set ) ) );
+                entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( set ) ) );
             }
             else if ( f instanceof EntityObjectField ) {
                 EntityObject eo = ( EntityObject ) field.getValue();
@@ -310,9 +323,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
                 locMap.put( "lon", locField.getValue().getLongitude() );
                 entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
             }
-            else if ( f instanceof DoubleField 
-                    || f instanceof FloatField 
-                    || f instanceof IntegerField
+            else if ( f instanceof DoubleField || f instanceof FloatField || f instanceof IntegerField
                     || f instanceof LongField ) {
 
                 entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue() );
@@ -323,7 +334,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
             }
             else if ( f instanceof UUIDField ) {
 
-                entityMap.put( STRING_PREFIX + field.getName().toLowerCase(), 
+                entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
                         field.getValue().toString().toLowerCase() );
             }
             else {
@@ -335,7 +346,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
     }
 
 
-    private static Collection processCollectionForMap( Collection c ) {
+    private static Collection processCollectionForMap( final Collection c ) {
         if ( c.isEmpty() ) {
             return c;
         }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index efefa8f..7378ecd 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -26,8 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
 import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
 import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -35,14 +33,17 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermFilterBuilder;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
@@ -53,6 +54,8 @@ import org.elasticsearch.search.sort.SortOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import org.apache.lucene.queryparser.xml.FilterBuilderFactory;
+
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.util.Health;
 import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -60,6 +63,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.IndexFig;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
@@ -228,29 +232,43 @@ public class EsEntityIndexImpl implements EntityIndex {
 
 
     @Override
-    public CandidateResults search( final IndexScope indexScope, final Query query ) {
+    public CandidateResults search( final IndexScope indexScope, final SearchTypes searchTypes, final Query query ) {
 
-        final String indexType = IndexingUtils.createCollectionScopeTypeName( indexScope );
+        final String context = IndexingUtils.createContextName( indexScope );
+        final String[] entityTypes = searchTypes.getTypeNames();
 
         QueryBuilder qb = query.createQueryBuilder();
 
         if ( logger.isDebugEnabled() ) {
-            logger.debug( "Searching index {}\n   type {}\n   query {} limit {}", new Object[] {
-                    this.indexName, indexType, qb.toString().replace( "\n", " " ), query.getLimit()
+            logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
+                    this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
             } );
         }
 
         SearchResponse searchResponse;
-        if ( query.getCursor() == null ) {
 
+        if ( query.getCursor() == null ) {
             SearchRequestBuilder srb =
-                    esProvider.getClient().prepareSearch( indexName ).setTypes( indexType ).setScroll( cursorTimeout + "m" )
+                    esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes ).setScroll(
+                            cursorTimeout + "m" )
                           .setQuery( qb );
 
-            FilterBuilder fb = query.createFilterBuilder();
+            final TermFilterBuilder contextFilter = FilterBuilders.termFilter( IndexingUtils.ENTITY_CONTEXT, context );
+
+            final FilterBuilder fb = query.createFilterBuilder();
+
+
+
+            //we have post filters, apply them
             if ( fb != null ) {
+//                final FilterBuilder postFilters = FilterBuilders.andFilter(fb, contextFilter  );
+
                 logger.debug( "   Filter: {} ", fb.toString() );
-                srb = srb.setPostFilter( fb );
+//                srb = srb.setPostFilter( postFilters );
+            }
+            //no other post filters, just the types
+            else{
+//                srb.setPostFilter( contextFilter );
             }
 
             srb = srb.setFrom( 0 ).setSize( query.getLimit() );
@@ -384,7 +402,7 @@ public class EsEntityIndexImpl implements EntityIndex {
     public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
         Query query = new Query();
         query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
-        CandidateResults results = search( scope, query );
+        CandidateResults results = search( scope, SearchTypes.fromTypes( id.getType() ), query );
         return results;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
index 2bf9bb4..13002e3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
@@ -25,12 +25,12 @@ import org.apache.usergrid.persistence.model.entity.Id;
 
 public class IndexScopeImpl implements IndexScope {
     private final Id ownerId;
-    private final String type;
+    private final String name;
 
 
-    public IndexScopeImpl( final Id ownerId, final String type ) {
+    public IndexScopeImpl( final Id ownerId, final String name ) {
         this.ownerId = ownerId;
-        this.type = type;
+        this.name = name;
 
         IndexValidationUtils.validateIndexScope( this );
     }
@@ -38,7 +38,7 @@ public class IndexScopeImpl implements IndexScope {
 
     @Override
     public String getName() {
-        return  type;
+        return  name;
     }
 
 
@@ -62,7 +62,7 @@ public class IndexScopeImpl implements IndexScope {
         if ( !ownerId.equals( that.ownerId ) ) {
             return false;
         }
-        if ( !type.equals( that.type ) ) {
+        if ( !name.equals( that.name ) ) {
             return false;
         }
 
@@ -73,7 +73,7 @@ public class IndexScopeImpl implements IndexScope {
     @Override
     public int hashCode() {
         int result = ownerId.hashCode();
-        result = 31 * result + type.hashCode();
+        result = 31 * result + name.hashCode();
         return result;
     }
 
@@ -82,7 +82,7 @@ public class IndexScopeImpl implements IndexScope {
     public String toString() {
         return "IndexScopeImpl{" +
                 "ownerId=" + ownerId +
-                ", type='" + type + '\'' +
+                ", name='" + name + '\'' +
                 '}';
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index f3e1ba2..c8e0b86 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -37,24 +37,30 @@ public class IndexingUtils {
     public static final String NUMBER_PREFIX = "nu_";
     public static final String BOOLEAN_PREFIX = "bu_";
 
-    public static final String ENTITYID_FIELDNAME = "zzz_entityid_zzz";
+    public static final String ENTITYID_FIELDNAME = "entityId";
 
     public static final String DOC_ID_SEPARATOR = "|";
     public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
 
     // These are not allowed in document type names: _ . , | #
     public static final String DOC_TYPE_SEPARATOR = "^";
-    public static final String DOC_TYPE_SEPARATOR_SPLITTER = "\\^";
 
     public static final String INDEX_NAME_SEPARATOR = "^";
 
+    public static final String ENTITY_CONTEXT = "_context";
+
+    /**
+     * To be used when we want to search all types within a scope
+     */
+    public static final String ALL_TYPES = "ALL";
+
 
     /**
       * Create our sub scope.  This is the ownerUUID + type
       * @param scope
       * @return
       */
-     public static String createCollectionScopeTypeName( IndexScope scope ) {
+     public static String createContextName( IndexScope scope ) {
          StringBuilder sb = new StringBuilder();
          sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR);
          sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR);
@@ -86,8 +92,8 @@ public class IndexingUtils {
      * @param entity
      * @return
      */
-    public static String createIndexDocId(Entity entity) {
-        return createIndexDocId(entity.getId(), entity.getVersion());
+    public static String createIndexDocId(final Entity entity, final String scopeType) {
+        return createIndexDocId(entity.getId(), entity.getVersion(), scopeType);
     }
 
 
@@ -97,11 +103,12 @@ public class IndexingUtils {
      * @param version
      * @return
      */
-    public static String createIndexDocId(Id entityId, UUID version) {
+    public static String createIndexDocId(final Id entityId, final UUID version, final String scopeType) {
         StringBuilder sb = new StringBuilder();
         sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR);
         sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR);
-        sb.append( version.toString() );
+        sb.append( version.toString() ).append( DOC_ID_SEPARATOR );
+        sb.append( scopeType);
         return sb.toString();
     }
 
@@ -117,7 +124,7 @@ public class IndexingUtils {
      *
      * @throws java.io.IOException On JSON generation error.
      */
-    public static XContentBuilder createDoubleStringIndexMapping( 
+    public static XContentBuilder createDoubleStringIndexMapping(
             XContentBuilder builder, String type ) throws IOException {
 
         builder = builder
@@ -142,6 +149,7 @@ public class IndexingUtils {
                             // all other strings are not analyzed
                         .startObject()
                             .startObject( "template_2" )
+                //todo, should be string prefix, remove 2 field mapping
                                 .field( "match", "*" )
                                 .field( "match_mapping_type", "string" )
                                 .startObject( "mapping" )
@@ -161,6 +169,17 @@ public class IndexingUtils {
                             .endObject()
                         .endObject()
 
+                //types for context direct string matching
+                .startObject( "context_template" )
+                        .field( "match", IndexingUtils.ENTITY_CONTEXT )
+                        .field( "match_mapping_type", "string" )
+                        .startObject( "mapping" )
+                            .field( "type", "string" )
+                            .field( "index", "not_analyzed" )
+                        .endObject()
+                    .endObject()
+                .endObject()
+
                     .endArray()
 
                 .endObject()

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index 0d7b0d8..2f71e2e 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -17,8 +17,7 @@
  */
 package org.apache.usergrid.persistence.index.impl;
 
-import com.google.inject.Guice;
-import com.google.inject.Injector;
+
 import java.io.BufferedReader;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
@@ -26,7 +25,15 @@ import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.List;
+
+import org.junit.ClassRule;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.commons.lang3.math.NumberUtils;
+
 import org.apache.usergrid.persistence.collection.CollectionScope;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -38,6 +45,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.guice.TestIndexModule;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.EntityResults;
@@ -49,11 +57,9 @@ import org.apache.usergrid.persistence.model.field.DoubleField;
 import org.apache.usergrid.persistence.model.field.LongField;
 import org.apache.usergrid.persistence.model.field.StringField;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.junit.ClassRule;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
 
 
 
@@ -127,7 +133,7 @@ public class CorePerformanceIT extends BaseIT {
 
             String appName = "app-" + j + "-" + time;
             Id appId = new SimpleId( appName );
-            IndexScope indexScope = new IndexScopeImpl( appId, "reviews" );
+            IndexScope indexScope = new IndexScopeImpl( appId, "reviews");
             scopes.add( indexScope );
 
             Thread t = new Thread( new DataLoader( applicationScope, indexScope ) );
@@ -178,12 +184,14 @@ public class CorePerformanceIT extends BaseIT {
             Query query = Query.fromQL( "review_score > 0"); // get all reviews;
             query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
 
-            CandidateResults candidateResults = eci.search(indexScope,  query );
+            final SearchTypes searchType = SearchTypes.fromTypes( "review" );
+
+            CandidateResults candidateResults = eci.search(indexScope, searchType, query );
             int count = candidateResults.size();
 
             while ( candidateResults.hasCursor() && count < maxEntities ) {
                 query.setCursor( candidateResults.getCursor() )   ;
-                candidateResults = eci.search(indexScope,  query );
+                candidateResults = eci.search(indexScope, searchType,  query );
                 count += candidateResults.size();
 
                 //cause retrieval from cassandra
@@ -308,10 +316,6 @@ public class CorePerformanceIT extends BaseIT {
     public void runSelectedQueries(final ApplicationScope scope,  List<IndexScope> indexScopes ) {
 
         for ( IndexScope indexScope : indexScopes ) {
-
-
-            CollectionScope collectionScope = new CollectionScopeImpl(
-                    scope.getApplication(), indexScope.getOwner(), indexScope.getName() );
             EntityIndex eci = ecif.createEntityIndex(scope );
 
             // TODO: come up with more and more complex queries for CorePerformanceIT
@@ -332,8 +336,8 @@ public class CorePerformanceIT extends BaseIT {
 
     public static void query(final IndexScope indexScope, final EntityIndex eci, final String query ) {;
         Query q = Query.fromQL(query) ;
-        CandidateResults candidateResults = eci.search(indexScope,  q );
-        log.info("size = {} returned from query {}", candidateResults.size(), q.getQl() );
+//        CandidateResults candidateResults = eci.search(indexScope,  q );  TODO FIXME
+//        log.info("size = {} returned from query {}", candidateResults.size(), q.getQl() );
     }
 
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 7ce230b..c1c075e 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -21,7 +21,6 @@ package org.apache.usergrid.persistence.index.impl;
 import java.io.IOException;
 import java.util.HashMap;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
@@ -29,17 +28,17 @@ import org.junit.runner.RunWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.usergrid.persistence.collection.CollectionScope;
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
 import org.apache.usergrid.persistence.collection.util.EntityUtils;
 import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.guice.TestIndexModule;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
@@ -52,7 +51,6 @@ import com.google.inject.Inject;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
 
 
 @RunWith( EsRunner.class )
@@ -74,6 +72,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
     @Inject
     public EntityIndexFactory ecif;
 
+
     @Test
     public void testBasicOperation() throws IOException {
 
@@ -81,47 +80,123 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
 
         // create a muffin
-        CollectionScope muffinScope = new CollectionScopeImpl( appId, appId, "muffins" );
-        Entity muffin = new Entity( 
-                new SimpleId( UUIDGenerator.newTimeUUID(), muffinScope.getName() ) );
+        Entity muffin = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "muffin" ) );
 
         muffin = EntityIndexMapUtils.fromMap( muffin, new HashMap<String, Object>() {{
             put( "size", "Large" );
             put( "flavor", "Blueberry" );
+            put( "stars", 5 );
         }} );
         EntityUtils.setVersion( muffin, UUIDGenerator.newTimeUUID() );
 
+        Entity egg = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "egg" ) );
 
-        // create a person who likes muffins
-        CollectionScope peopleScope = new CollectionScopeImpl( appId, appId, "people" );
-        Entity person = new Entity( new SimpleId( 
-                UUIDGenerator.newTimeUUID(), peopleScope.getName() ) );
-        person = EntityIndexMapUtils.fromMap( person, new HashMap<String, Object>() {{
-            put( "name", "Dave" );
-            put( "hometown", "Chapel Hill" );
+        egg = EntityIndexMapUtils.fromMap( egg, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "type", "scramble" );
+            put( "stars", 5 );
+        }} );
+        EntityUtils.setVersion( egg, UUIDGenerator.newTimeUUID() );
+
+        Entity oj = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "juice" ) );
+
+        oj = EntityIndexMapUtils.fromMap( oj, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "type", "pulpy" );
+            put( "stars", 3 );
         }} );
-        EntityUtils.setVersion( person, UUIDGenerator.newTimeUUID() );
+        EntityUtils.setVersion( oj, UUIDGenerator.newTimeUUID() );
+
+
+        // create a person who likes muffins
+        Id personId =   new SimpleId( UUIDGenerator.newTimeUUID(), "person" ) ;
 
 
-        assertNotNull( person.getId() );
-        assertNotNull( person.getId().getUuid() );
+        assertNotNull( personId );
+        assertNotNull( personId.getType() );
+        assertNotNull( personId.getUuid() );
 
         // index connection of "person Dave likes Large Blueberry muffin"
 
-        IndexScope scope = new IndexScopeImpl(  person.getId(), "likes" );
+        IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
+
+        //create another scope we index in, want to be sure these scopes are filtered
+        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "person" ), "likes" );
 
         EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
 
         EntityIndexBatch batch = personLikesIndex.createBatch();
 
-        batch.index( scope, muffin );
+        //add to both scopes
+
+        //add a muffin
+        batch.index( searchScope, muffin );
+        batch.index( otherIndexScope, muffin );
+
+        //add the eggs
+        batch.index( searchScope, egg );
+        batch.index( otherIndexScope, egg );
+
+        //add the oj
+        batch.index( searchScope, oj );
+        batch.index( otherIndexScope, oj );
+
         batch.executeAndRefresh();
 
-        // now, let's search for things that Dave likes
-        CandidateResults likes = personLikesIndex.search(scope,  Query.fromQL( "select *" ) );
+        // now, let's search for muffins
+        CandidateResults likes = personLikesIndex
+                .search( searchScope, SearchTypes.fromTypes( muffin.getId().getType() ), Query.fromQL( "select *" ) );
         assertEquals( 1, likes.size() );
-        assertEquals(muffin.getId(), likes.get(0).getId());
+        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
 
-    }
+        // now, let's search for egg
+        likes = personLikesIndex
+                .search( searchScope, SearchTypes.fromTypes( egg.getId().getType() ), Query.fromQL( "select *" ) );
+        assertEquals( 1, likes.size() );
+        assertEquals( egg.getId(), likes.get( 0 ).getId() );
+
+        // search for OJ
+        likes = personLikesIndex
+                .search( searchScope, SearchTypes.fromTypes( oj.getId().getType() ), Query.fromQL( "select *" ) );
+        assertEquals( 1, likes.size() );
+        assertEquals( oj.getId(), likes.get( 0 ).getId() );
+
+
+        //now lets search for all explicitly
+        likes = personLikesIndex.search( searchScope,
+                SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
+                Query.fromQL( "select *" ) );
+        assertEquals( 3, likes.size() );
+        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
+        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+        assertEquals( oj.getId(), likes.get( 2 ).getId() );
+
+          //now lets search for all explicitly
+        likes = personLikesIndex.search( searchScope,
+                SearchTypes.allTypes(),
+                Query.fromQL( "select *" ) );
+        assertEquals( 3, likes.size() );
+        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
+        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+        assertEquals( oj.getId(), likes.get( 2 ).getId() );
 
+
+
+        //now search all entity types with a query that returns a subset
+        likes = personLikesIndex.search( searchScope,
+                SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
+                Query.fromQL( "select * where stars = 5" ) );
+        assertEquals( 2, likes.size() );
+        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
+        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+
+
+
+        //now search with no types, we should get only the results that match
+        likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where stars = 5" ) );
+        assertEquals( 2, likes.size() );
+        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
+        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2db132c3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 3be996e..1bbb301 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -59,7 +59,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Maps;
 import com.google.inject.Inject;
-import java.util.ArrayList;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -70,393 +69,393 @@ import static org.junit.Assert.assertNotEquals;
 @UseModules({ TestIndexModule.class })
 public class EntityIndexTest extends BaseIT {
 
-    private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
-
-    @ClassRule
-    public static CassandraRule cass = new CassandraRule();
-
-    @Rule
-    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
-
-    @Inject
-    @Rule
-    public MigrationManagerRule migrationManagerRule;
-
-    @Inject
-    public EntityIndexFactory eif;
-
-
-
-    @Test
-    public void testIndex() throws IOException {
-
-        final int MAX_ENTITIES = 100;
-
-        Id appId = new SimpleId( "application" );
-
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
-
-
-        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-        entityIndex.initializeIndex();
-
-        InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
-        ObjectMapper mapper = new ObjectMapper();
-        List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
-
-        int count = 0;
-        StopWatch timer = new StopWatch();
-        timer.start();
-
-        final EntityIndexBatch batch = entityIndex.createBatch();
-
-        for ( Object o : sampleJson ) {
-
-            Map<String, Object> item = ( Map<String, Object> ) o;
-
-            Entity entity = new Entity( indexScope.getName() );
-            entity = EntityIndexMapUtils.fromMap( entity, item );
-            EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-
-            batch.index( indexScope, entity );
-
-            if(count %1000 == 0){
-                batch.execute();
-            }
-
-
-
-            if ( count++ > MAX_ENTITIES ) {
-                break;
-            }
-        }
-
-        batch.execute();
-        timer.stop();
-        log.info( "Total time to index {} entries {}ms, average {}ms/entry",
-                new Object[] { count, timer.getTime(), timer.getTime() / count } );
-
-        entityIndex.refresh();
-
-
-        testQueries( indexScope, entityIndex );
-    }
-
-
-    @Test
-    public void testDeindex() {
-
-        Id appId = new SimpleId( "application" );
-
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
-
-        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-        entityIndex.initializeIndex();
-
-        Map entityMap = new HashMap() {{
-            put( "name", "Ferrari 212 Inter" );
-            put( "introduced", 1952 );
-            put( "topspeed", 215 );
-        }};
-
-
-        Entity entity = EntityIndexMapUtils.fromMap( entityMap );
-        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
-        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-        entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
-
-        CandidateResults candidateResults = entityIndex.search(indexScope,  Query.fromQL( "name contains 'Ferrari*'" ) );
-        assertEquals( 1, candidateResults.size() );
-
-        entityIndex.createBatch().deindex( indexScope, entity ).execute();
-
-        entityIndex.refresh();
-
-        candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) );
-        assertEquals( 0, candidateResults.size() );
-    }
-
-
-    private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) {
-
-        StopWatch timer = new StopWatch();
-        timer.start();
-        Query query = Query.fromQL( queryString );
-        query.setLimit( 1000 );
-        CandidateResults candidateResults = entityIndex.search( scope, query );
-        timer.stop();
-
-        assertEquals( num, candidateResults.size() );
-        log.debug( "Query time {}ms", timer.getTime() );
-    }
-
-
-    private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) {
-
-
-        testQuery(scope,  entityIndex, "name = 'Morgan Pierce'", 1 );
-
-        testQuery(scope,  entityIndex, "name = 'morgan pierce'", 1 );
-
-        testQuery(scope,  entityIndex, "name = 'Morgan'", 0 );
-
-        testQuery(scope,  entityIndex, "name contains 'Morgan'", 1 );
-
-        testQuery(scope,  entityIndex, "company > 'GeoLogix'", 64 );
-
-        testQuery(scope,  entityIndex, "gender = 'female'", 45 );
-
-        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
-
-        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
-
-        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
-
-        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
-
-        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
-        
-        testQuery(scope,  entityIndex, "name = 'Morgan* '", 1 );
-        
-        testQuery(scope,  entityIndex, "name = 'Morgan*'", 1 );
-
-        
-        // test a couple of array sub-property queries
-
-        int totalUsers = 102;
-
-        // nobody has a friend named Jack the Ripper
-        testQuery(scope,  entityIndex, "friends.name = 'Jack the Ripper'", 0 );
-
-        // everybody doesn't have a friend named Jack the Ripper
-        testQuery(scope,  entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
-
-        // one person has a friend named Shari Hahn
-        testQuery(scope,  entityIndex, "friends.name = 'Wendy Moody'", 1 );
-
-        // everybody but 1 doesn't have a friend named Shari Hahh
-        testQuery(scope,  entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
-
-    }
-
-
-    /**
-     * Tests that Entity-to-map and Map-to-entity round trip works.
-     */
-    @Test
-    public void testEntityIndexMapUtils() throws IOException {
-
-        InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" );
-        ObjectMapper mapper = new ObjectMapper();
-        List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} );
-
-        for ( Object o : contacts ) {
-
-            Map<String, Object> map1 = ( Map<String, Object> ) o;
-
-            // convert map to entity
-            Entity entity1 = EntityIndexMapUtils.fromMap( map1 );
-
-            // convert entity back to map
-            Map map2 = EntityIndexMapUtils.toMap( entity1 );
-
-            // the two maps should be the same 
-            Map diff = Maps.difference( map1, map2 ).entriesDiffering();
-            assertEquals( 0, diff.size() );
-        }
-    }
-
-
-    @Test
-    public void getEntityVersions() throws Exception {
-
-        Id appId = new SimpleId( "application" );
-        Id ownerId = new SimpleId( "owner" );
-
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        IndexScope indexScope = new IndexScopeImpl( ownerId, "user" );
-
-
-
-        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-        entityIndex.initializeIndex();
-
-        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-        Map<String, Object> properties = new LinkedHashMap<String, Object>();
-        properties.put( "username", "edanuff" );
-        properties.put( "email", "ed@anuff.com" );
-        properties.put( "middlename", middleName );
-
-        Map entityMap = new HashMap() {{
-            put( "username", "edanuff" );
-            put( "email", "ed@anuff.com" );
-            put( "middlename", middleName );
-        }};
-
-        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
-        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-
-
-        final EntityIndexBatch batch = entityIndex.createBatch();
-
-        batch.index( indexScope, user );
-
-        user.setField( new StringField( "address1", "1782 address st" ) );
-        batch.index( indexScope, user );
-        user.setField( new StringField( "address2", "apt 508" ) );
-        batch.index( indexScope,  user );
-        user.setField( new StringField( "address3", "apt 508" ) );
-        batch.index( indexScope,  user );
-        batch.executeAndRefresh();
-
-        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId() );
-
-        assertEquals(1,  results.size());
-        assertEquals( results.get( 0 ).getId(), user.getId() );
-        assertEquals( results.get(0).getVersion(), user.getVersion());
-    }
-
-
-    @Test
-    public void deleteVerification() throws Throwable {
-
-        Id appId = new SimpleId( "application" );
-        Id ownerId = new SimpleId( "owner" );
-
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
-
-        EntityIndex ei = eif.createEntityIndex( applicationScope );
-        ei.initializeIndex();
-
-        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-
-        Map entityMap = new HashMap() {{
-            put( "username", "edanuff" );
-            put( "email", "ed@anuff.com" );
-            put( "middlename", middleName );
-        }};
-
-        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
-        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-
-
-        EntityIndexBatch batch = ei.createBatch();
-
-        batch.index( appScope, user ).executeAndRefresh();
-        Query query = new Query();
-        query.addEqualityFilter( "username", "edanuff" );
-        CandidateResults r = ei.search( appScope, query );
-        assertEquals( user.getId(), r.get( 0 ).getId() );
-
-        batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
-
-
-        // EntityRef
-        query = new Query();
-        query.addEqualityFilter( "username", "edanuff" );
-        r = ei.search(appScope, query );
-
-        assertFalse( r.iterator().hasNext() );
-    }
-
-    @Test 
-    public void multiValuedTypes() {
-
-        Id appId = new SimpleId( "entityindextest" );
-        Id ownerId = new SimpleId( "multivaluedtype" );
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
-
-        EntityIndex ei = eif.createEntityIndex( applicationScope );
-        ei.createBatch();
-
-        // Bill has favorites as string, age as string and retirement goal as number
-        Map billMap = new HashMap() {{
-            put( "username", "bill" );
-            put( "email", "bill@example.com" );
-            put( "age", "thirtysomething");
-            put( "favorites", "scallops, croquet, wine");
-            put( "retirementGoal", 100000);
-        }};
-        Entity bill = EntityIndexMapUtils.fromMap( billMap );
-        EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
-        EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
-
-        EntityIndexBatch batch = ei.createBatch();
-
-        batch.index( appScope,  bill );
-
-        // Fred has age as int, favorites as object and retirement goal as object
-        Map fredMap = new HashMap() {{
-            put( "username", "fred" );
-            put( "email", "fred@example.com" );
-            put( "age", 41 );
-            put( "favorites", new HashMap<String, Object>() {{
-                put("food", "cheezewiz"); 
-                put("sport", "nascar"); 
-                put("beer", "budwizer"); 
-            }});
-            put( "retirementGoal", new HashMap<String, Object>() {{
-                put("car", "Firebird"); 
-                put("home", "Mobile"); 
-            }});
-        }};
-        Entity fred = EntityIndexMapUtils.fromMap( fredMap );
-        EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
-        EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
-        batch.index( appScope, fred );
-
-        batch.executeAndRefresh();
-
-        Query query = new Query();
-        query.addEqualityFilter( "username", "bill" );
-        CandidateResults r = ei.search( appScope, query );
-        assertEquals( bill.getId(), r.get( 0 ).getId() );
-
-        query = new Query();
-        query.addEqualityFilter( "username", "fred" );
-        r = ei.search( appScope,  query );
-        assertEquals( fred.getId(), r.get( 0 ).getId() );
-
-        query = new Query();
-        query.addEqualityFilter( "age", 41 );
-        r = ei.search( appScope,  query );
-        assertEquals( fred.getId(), r.get( 0 ).getId() );
-
-        query = new Query();
-        query.addEqualityFilter( "age", "thirtysomething" );
-        r = ei.search(  appScope, query );
-        assertEquals( bill.getId(), r.get( 0 ).getId() );
-    }
-
-    
-    @Test
-    public void healthTest() {
-
-        Id appId = new SimpleId( "application" );
-        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
-        EntityIndex ei = eif.createEntityIndex( applicationScope ); 
-
-        assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
-        assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() );
-
-        ei.initializeIndex();
-        ei.refresh();
-
-        assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
-        assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
-    }
+//    private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
+//
+//    @ClassRule
+//    public static CassandraRule cass = new CassandraRule();
+//
+//    @Rule
+//    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
+//
+//    @Inject
+//    @Rule
+//    public MigrationManagerRule migrationManagerRule;
+//
+//    @Inject
+//    public EntityIndexFactory eif;
+//
+//
+//
+//    @Test
+//    public void testIndex() throws IOException {
+//
+//        final int MAX_ENTITIES = 100;
+//
+//        Id appId = new SimpleId( "application" );
+//
+//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+//
+//        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+//
+//
+//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+//        entityIndex.initializeIndex();
+//
+//        InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
+//        ObjectMapper mapper = new ObjectMapper();
+//        List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
+//
+//        int count = 0;
+//        StopWatch timer = new StopWatch();
+//        timer.start();
+//
+//        final EntityIndexBatch batch = entityIndex.createBatch();
+//
+//        for ( Object o : sampleJson ) {
+//
+//            Map<String, Object> item = ( Map<String, Object> ) o;
+//
+//            Entity entity = new Entity( indexScope.getName() );
+//            entity = EntityIndexMapUtils.fromMap( entity, item );
+//            EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+//
+//            batch.index( indexScope, entity );
+//
+//            if(count %1000 == 0){
+//                batch.execute();
+//            }
+//
+//
+//
+//            if ( count++ > MAX_ENTITIES ) {
+//                break;
+//            }
+//        }
+//
+//        batch.execute();
+//        timer.stop();
+//        log.info( "Total time to index {} entries {}ms, average {}ms/entry",
+//                new Object[] { count, timer.getTime(), timer.getTime() / count } );
+//
+//        entityIndex.refresh();
+//
+//
+//        testQueries( indexScope, entityIndex );
+//    }
+//
+//
+//    @Test
+//    public void testDeindex() {
+//
+//        Id appId = new SimpleId( "application" );
+//
+//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+//
+//        IndexScope indexScope = new IndexScopeImpl( appId, "fastcars", entityType );
+//
+//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+//        entityIndex.initializeIndex();
+//
+//        Map entityMap = new HashMap() {{
+//            put( "name", "Ferrari 212 Inter" );
+//            put( "introduced", 1952 );
+//            put( "topspeed", 215 );
+//        }};
+//
+//
+//        Entity entity = EntityIndexMapUtils.fromMap( entityMap );
+//        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
+//        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+//        entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
+//
+//        CandidateResults candidateResults = entityIndex.search(indexScope,  Query.fromQL( "name contains 'Ferrari*'" ) );
+//        assertEquals( 1, candidateResults.size() );
+//
+//        entityIndex.createBatch().deindex( indexScope, entity ).execute();
+//
+//        entityIndex.refresh();
+//
+//        candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) );
+//        assertEquals( 0, candidateResults.size() );
+//    }
+//
+//
+//    private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) {
+//
+//        StopWatch timer = new StopWatch();
+//        timer.start();
+//        Query query = Query.fromQL( queryString );
+//        query.setLimit( 1000 );
+//        CandidateResults candidateResults = entityIndex.search( scope, query );
+//        timer.stop();
+//
+//        assertEquals( num, candidateResults.size() );
+//        log.debug( "Query time {}ms", timer.getTime() );
+//    }
+//
+//
+//    private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) {
+//
+//
+//        testQuery(scope,  entityIndex, "name = 'Morgan Pierce'", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'morgan pierce'", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Morgan'", 0 );
+//
+//        testQuery(scope,  entityIndex, "name contains 'Morgan'", 1 );
+//
+//        testQuery(scope,  entityIndex, "company > 'GeoLogix'", 64 );
+//
+//        testQuery(scope,  entityIndex, "gender = 'female'", 45 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Morgan* '", 1 );
+//
+//        testQuery(scope,  entityIndex, "name = 'Morgan*'", 1 );
+//
+//
+//        // test a couple of array sub-property queries
+//
+//        int totalUsers = 102;
+//
+//        // nobody has a friend named Jack the Ripper
+//        testQuery(scope,  entityIndex, "friends.name = 'Jack the Ripper'", 0 );
+//
+//        // everybody doesn't have a friend named Jack the Ripper
+//        testQuery(scope,  entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
+//
+//        // one person has a friend named Shari Hahn
+//        testQuery(scope,  entityIndex, "friends.name = 'Wendy Moody'", 1 );
+//
+//        // everybody but 1 doesn't have a friend named Shari Hahh
+//        testQuery(scope,  entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
+//
+//    }
+//
+//
+//    /**
+//     * Tests that Entity-to-map and Map-to-entity round trip works.
+//     */
+//    @Test
+//    public void testEntityIndexMapUtils() throws IOException {
+//
+//        InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" );
+//        ObjectMapper mapper = new ObjectMapper();
+//        List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} );
+//
+//        for ( Object o : contacts ) {
+//
+//            Map<String, Object> map1 = ( Map<String, Object> ) o;
+//
+//            // convert map to entity
+//            Entity entity1 = EntityIndexMapUtils.fromMap( map1 );
+//
+//            // convert entity back to map
+//            Map map2 = EntityIndexMapUtils.toMap( entity1 );
+//
+//            // the two maps should be the same
+//            Map diff = Maps.difference( map1, map2 ).entriesDiffering();
+//            assertEquals( 0, diff.size() );
+//        }
+//    }
+//
+//
+//    @Test
+//    public void getEntityVersions() throws Exception {
+//
+//        Id appId = new SimpleId( "application" );
+//        Id ownerId = new SimpleId( "owner" );
+//
+//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+//
+//        IndexScope indexScope = new IndexScopeImpl( ownerId, "user", entityType );
+//
+//
+//
+//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+//        entityIndex.initializeIndex();
+//
+//        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+//        Map<String, Object> properties = new LinkedHashMap<String, Object>();
+//        properties.put( "username", "edanuff" );
+//        properties.put( "email", "ed@anuff.com" );
+//        properties.put( "middlename", middleName );
+//
+//        Map entityMap = new HashMap() {{
+//            put( "username", "edanuff" );
+//            put( "email", "ed@anuff.com" );
+//            put( "middlename", middleName );
+//        }};
+//
+//        Entity user = EntityIndexMapUtils.fromMap( entityMap );
+//        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
+//        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+//
+//
+//        final EntityIndexBatch batch = entityIndex.createBatch();
+//
+//        batch.index( indexScope, user );
+//
+//        user.setField( new StringField( "address1", "1782 address st" ) );
+//        batch.index( indexScope, user );
+//        user.setField( new StringField( "address2", "apt 508" ) );
+//        batch.index( indexScope,  user );
+//        user.setField( new StringField( "address3", "apt 508" ) );
+//        batch.index( indexScope,  user );
+//        batch.executeAndRefresh();
+//
+//        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId() );
+//
+//        assertEquals(1,  results.size());
+//        assertEquals( results.get( 0 ).getId(), user.getId() );
+//        assertEquals( results.get(0).getVersion(), user.getVersion());
+//    }
+//
+//
+//    @Test
+//    public void deleteVerification() throws Throwable {
+//
+//        Id appId = new SimpleId( "application" );
+//        Id ownerId = new SimpleId( "owner" );
+//
+//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+//
+//        IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
+//
+//        EntityIndex ei = eif.createEntityIndex( applicationScope );
+//        ei.initializeIndex();
+//
+//        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+//
+//        Map entityMap = new HashMap() {{
+//            put( "username", "edanuff" );
+//            put( "email", "ed@anuff.com" );
+//            put( "middlename", middleName );
+//        }};
+//
+//        Entity user = EntityIndexMapUtils.fromMap( entityMap );
+//        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
+//        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+//
+//
+//        EntityIndexBatch batch = ei.createBatch();
+//
+//        batch.index( appScope, user ).executeAndRefresh();
+//        Query query = new Query();
+//        query.addEqualityFilter( "username", "edanuff" );
+//        CandidateResults r = ei.search( appScope, query );
+//        assertEquals( user.getId(), r.get( 0 ).getId() );
+//
+//        batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
+//
+//
+//        // EntityRef
+//        query = new Query();
+//        query.addEqualityFilter( "username", "edanuff" );
+//        r = ei.search(appScope, query );
+//
+//        assertFalse( r.iterator().hasNext() );
+//    }
+//
+//    @Test
+//    public void multiValuedTypes() {
+//
+//        Id appId = new SimpleId( "entityindextest" );
+//        Id ownerId = new SimpleId( "multivaluedtype" );
+//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+//
+//        IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
+//
+//        EntityIndex ei = eif.createEntityIndex( applicationScope );
+//        ei.createBatch();
+//
+//        // Bill has favorites as string, age as string and retirement goal as number
+//        Map billMap = new HashMap() {{
+//            put( "username", "bill" );
+//            put( "email", "bill@example.com" );
+//            put( "age", "thirtysomething");
+//            put( "favorites", "scallops, croquet, wine");
+//            put( "retirementGoal", 100000);
+//        }};
+//        Entity bill = EntityIndexMapUtils.fromMap( billMap );
+//        EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
+//        EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
+//
+//        EntityIndexBatch batch = ei.createBatch();
+//
+//        batch.index( appScope,  bill );
+//
+//        // Fred has age as int, favorites as object and retirement goal as object
+//        Map fredMap = new HashMap() {{
+//            put( "username", "fred" );
+//            put( "email", "fred@example.com" );
+//            put( "age", 41 );
+//            put( "favorites", new HashMap<String, Object>() {{
+//                put("food", "cheezewiz");
+//                put("sport", "nascar");
+//                put("beer", "budwizer");
+//            }});
+//            put( "retirementGoal", new HashMap<String, Object>() {{
+//                put("car", "Firebird");
+//                put("home", "Mobile");
+//            }});
+//        }};
+//        Entity fred = EntityIndexMapUtils.fromMap( fredMap );
+//        EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
+//        EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
+//        batch.index( appScope, fred );
+//
+//        batch.executeAndRefresh();
+//
+//        Query query = new Query();
+//        query.addEqualityFilter( "username", "bill" );
+//        CandidateResults r = ei.search( appScope, query );
+//        assertEquals( bill.getId(), r.get( 0 ).getId() );
+//
+//        query = new Query();
+//        query.addEqualityFilter( "username", "fred" );
+//        r = ei.search( appScope,  query );
+//        assertEquals( fred.getId(), r.get( 0 ).getId() );
+//
+//        query = new Query();
+//        query.addEqualityFilter( "age", 41 );
+//        r = ei.search( appScope,  query );
+//        assertEquals( fred.getId(), r.get( 0 ).getId() );
+//
+//        query = new Query();
+//        query.addEqualityFilter( "age", "thirtysomething" );
+//        r = ei.search(  appScope, query );
+//        assertEquals( bill.getId(), r.get( 0 ).getId() );
+//    }
+//
+//
+//    @Test
+//    public void healthTest() {
+//
+//        Id appId = new SimpleId( "application" );
+//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+//
+//        EntityIndex ei = eif.createEntityIndex( applicationScope );
+//
+//        assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
+//        assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() );
+//
+//        ei.initializeIndex();
+//        ei.refresh();
+//
+//        assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
+//        assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
+//    }
 }
 
 


[10/27] incubator-usergrid git commit: Changed setup so that the first rest server performs migrations, system setup, then modifies the column families with the performance tweaks.

Posted by to...@apache.org.
Changed setup so that the first rest server performs migrations, system setup, then modifies the column families with the performance tweaks.

Fixes setup issue with keyspaces


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/54772ab0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/54772ab0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/54772ab0

Branch: refs/heads/two-dot-o
Commit: 54772ab0af08a9651d6fe166fbbee369adeb6a5f
Parents: 3495001
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 12:49:52 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 12:49:52 2014 -0700

----------------------------------------------------------------------
 stack/awscluster/src/main/cql/create_locks.cql  | 26 --------
 .../awscluster/src/main/cql/create_usergrid.cql | 28 ---------
 .../main/cql/create_usergrid_applications.cql   | 47 --------------
 stack/awscluster/src/main/cql/update_locks.cql  | 23 +++++++
 .../awscluster/src/main/cql/update_usergrid.cql | 27 ++++++++
 .../main/cql/update_usergrid_applications.cql   | 46 ++++++++++++++
 .../main/dist/init_instance/create_keyspaces.sh | 61 ------------------
 .../main/dist/init_instance/init_db_server.sh   |  4 --
 .../main/dist/init_instance/init_rest_server.sh | 20 +++++-
 .../main/dist/init_instance/update_keyspaces.sh | 66 ++++++++++++++++++++
 .../src/main/groovy/NodeRegistry.groovy         |  8 ++-
 stack/awscluster/ugcluster-cf.json              | 14 ++---
 .../usergrid/corepersistence/CpSetup.java       |  4 +-
 .../persistence/cassandra/CassandraService.java |  3 +
 14 files changed, 198 insertions(+), 179 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/cql/create_locks.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/create_locks.cql b/stack/awscluster/src/main/cql/create_locks.cql
deleted file mode 100644
index fc1de2a..0000000
--- a/stack/awscluster/src/main/cql/create_locks.cql
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.  For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-drop keyspace Locks;
-
-create keyspace Locks with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = false;
-update keyspace Locks with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = false;
-
-use Locks;
-
-create column family HLocks with gc_grace=60 and caching=ALL and read_repair_chance=0.00 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/cql/create_usergrid.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/create_usergrid.cql b/stack/awscluster/src/main/cql/create_usergrid.cql
deleted file mode 100644
index deae504..0000000
--- a/stack/awscluster/src/main/cql/create_usergrid.cql
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.  For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-create keyspace Usergrid with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
-update keyspace Usergrid with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
-
-use Usergrid;
-
-create column family Applications with caching=KEYS_ONLY and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family PrincipalTokens with caching=KEYS_ONLY and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Properties with caching=KEYS_ONLY and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Tokens with caching=ALL and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/cql/create_usergrid_applications.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/create_usergrid_applications.cql b/stack/awscluster/src/main/cql/create_usergrid_applications.cql
deleted file mode 100644
index bb3d29a..0000000
--- a/stack/awscluster/src/main/cql/create_usergrid_applications.cql
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- *  contributor license agreements.  The ASF licenses this file to You
- * under the Apache License, Version 2.0 (the "License"); you may not
- * use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.  For additional information regarding
- * copyright in this work, please see the NOTICE file in the top level
- * directory of this distribution.
- */
-
-create keyspace Usergrid_Applications with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
-update keyspace Usergrid_Applications with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
-
-use Usergrid_Applications;
-
-create column family Application_Aggregate_Counters with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Application_Roles with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Applications with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Consumer_Queue_Messages_Properties with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Composite_Dictionaries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Counters with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Dictionaries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Id_Sets with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Index with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Index_Entries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Metadata with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Properties with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Entity_Unique with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family MQ_Consumers with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family MQ_Consumers_Timeout with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family MQ_Counters with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family MQ_Property_Index with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family MQ_Property_Index_Entries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Queue_Dictionaries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Queue_Inbox with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Queue_Properties with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Queue_Subscribers with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-create column family Queue_Subscriptions with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
-

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/cql/update_locks.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/update_locks.cql b/stack/awscluster/src/main/cql/update_locks.cql
new file mode 100644
index 0000000..64344f6
--- /dev/null
+++ b/stack/awscluster/src/main/cql/update_locks.cql
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+update keyspace Locks with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = false;
+
+use Locks;
+
+update column family HLocks with gc_grace=60 and caching=ALL and read_repair_chance=0.00 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/cql/update_usergrid.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/update_usergrid.cql b/stack/awscluster/src/main/cql/update_usergrid.cql
new file mode 100644
index 0000000..50eb79f
--- /dev/null
+++ b/stack/awscluster/src/main/cql/update_usergrid.cql
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+update keyspace Usergrid with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
+
+use Usergrid;
+
+update column family Applications with caching=KEYS_ONLY and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family PrincipalTokens with caching=KEYS_ONLY and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Properties with caching=KEYS_ONLY and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Tokens with caching=ALL and read_repair_chance=0.1  and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/cql/update_usergrid_applications.cql
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/cql/update_usergrid_applications.cql b/stack/awscluster/src/main/cql/update_usergrid_applications.cql
new file mode 100644
index 0000000..253230d
--- /dev/null
+++ b/stack/awscluster/src/main/cql/update_usergrid_applications.cql
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  The ASF licenses this file to You
+ * under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.  For additional information regarding
+ * copyright in this work, please see the NOTICE file in the top level
+ * directory of this distribution.
+ */
+
+update keyspace Usergrid_Applications with placement_strategy = 'NetworkTopologyStrategy' and strategy_options = {KEYSPACE_REGION : 3} and durable_writes = true;
+
+use Usergrid_Applications;
+
+update column family Application_Aggregate_Counters with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Application_Roles with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Applications with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Consumer_Queue_Messages_Properties with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Composite_Dictionaries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Counters with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Dictionaries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Id_Sets with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Index with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Index_Entries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Metadata with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Properties with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Entity_Unique with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family MQ_Consumers with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family MQ_Consumers_Timeout with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family MQ_Counters with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family MQ_Property_Index with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family MQ_Property_Index_Entries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Queue_Dictionaries with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Queue_Inbox with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Queue_Properties with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Queue_Subscribers with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+update column family Queue_Subscriptions with caching=KEYS_ONLY and read_repair_chance=0.1 and compaction_strategy='LeveledCompactionStrategy' and bloom_filter_fp_chance=0.1 and compaction_strategy_options={sstable_size_in_mb:512};
+

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/dist/init_instance/create_keyspaces.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/create_keyspaces.sh b/stack/awscluster/src/main/dist/init_instance/create_keyspaces.sh
deleted file mode 100644
index e2451e8..0000000
--- a/stack/awscluster/src/main/dist/init_instance/create_keyspaces.sh
+++ /dev/null
@@ -1,61 +0,0 @@
-#!/bin/bash
-#
-#  Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements.  The ASF licenses this file to You
-#  under the Apache License, Version 2.0 (the "License"); you may not
-#  use this file except in compliance with the License.
-#  You may obtain a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-#  Unless required by applicable law or agreed to in writing, software
-#  distributed under the License is distributed on an "AS IS" BASIS,
-#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#  See the License for the specific language governing permissions and
-#  limitations under the License.  For additional information regarding
-#  copyright in this work, please see the NOTICE file in the top level
-#  directory of this distribution.
-#
-
-#Install keyspaces
-pushd /etc/apt/sources.list.d
-
-#Run the cassandra cql to create the keyspaces.  Note this only works for the
-# us-east region for the replication factor on the keyspaces
-
-cd /usr/share/usergrid/scripts
-FIRSTHOST="$(groovy get_first_instance.groovy cassandra)"
-
-if [ "$FIRSTHOST"=="$PUBLIC_HOSTNAME" ]; then
-
-#We have to wait for cass to actually start before we can run our CQL.   Sleep 5 seconds between retries
-while ! echo exit | nc localhost 9160; do sleep 5; done
-
-#WE have to remove our -1 from the end, since us-east and us-west dont support -1 in cassandra
-CASS_REGION=${EC2_REGION%-1}
-
-
-#Update the keyspace replication and run the cql
-sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/create_locks.cql
-
-/usr/bin/cassandra-cli -f  /usr/share/usergrid/cql/create_locks.cql
-
-
-
-
-#Update the keyspace region and run the cql
-sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/create_usergrid.cql
-
-/usr/bin/cassandra-cli -f  /usr/share/usergrid/cql/create_usergrid.cql
-
-
-
-
-#Update the keyspace region and run the cql
-sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/create_usergrid_applications.cql
-
-/usr/bin/cassandra-cli -f  /usr/share/usergrid/cql/create_usergrid_applications.cql
-
-fi
-
-popd

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_db_server.sh b/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
index a9bae98..eb8e1ba 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_db_server.sh
@@ -67,10 +67,6 @@ cd /usr/share/usergrid/init_instance
 cd /usr/share/usergrid/init_instance
 ./install_cassandra.sh
 
-# Use the CQL to crate the keyspaces
-cd /usr/share/usergrid/init_instance
-./create_keyspaces.sh
-
 # Install the opscenter agent
 cd /usr/share/usergrid/init_instance
 ./install_opscenter_agent.sh

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 1fe4bb8..7b6571c 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -206,12 +206,30 @@ sh /etc/init.d/tomcat7 start
 #Wait until tomcat starts and we can hit our status page
 until curl -m 1 -I -X GET http://localhost:8080/status | grep "200 OK";  do sleep 5; done
 
+
+#If we're the first rest server, run the migration, the database setup, then run the Cassanda keyspace updates
+cd /usr/share/usergrid/scripts
+groovy registry_register.groovy rest
+
+FIRSTHOST="$(groovy get_first_instance.groovy rest)"
+
+if [ "$FIRSTHOST"=="$PUBLIC_HOSTNAME" ]; then
+
 #Run the migration
 curl -X PUT http://localhost:8080/system/migrate/run  -u superuser:test
 
-#Run the system database setup
+#Wait since migration is no-op just needs to ideally run to completion to bring the internal state up to date before
+#Running setup
+sleep 10
+
+#Run the system database setup since migration is a no-op
 curl -X GET http://localhost:8080/system/database/setup -u superuser:test
 
+cd /usr/share/usergrid/init_instance
+./update_keyspaces.sh
+
+fi
+
 # tag last so we can see in the console that the script ran to completion
 cd /usr/share/usergrid/scripts
 groovy tag_instance.groovy

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh b/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
new file mode 100644
index 0000000..c309879
--- /dev/null
+++ b/stack/awscluster/src/main/dist/init_instance/update_keyspaces.sh
@@ -0,0 +1,66 @@
+#!/bin/bash
+#
+#  Licensed to the Apache Software Foundation (ASF) under one or more
+#   contributor license agreements.  The ASF licenses this file to You
+#  under the Apache License, Version 2.0 (the "License"); you may not
+#  use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.  For additional information regarding
+#  copyright in this work, please see the NOTICE file in the top level
+#  directory of this distribution.
+#
+
+#Install keyspaces
+pushd /etc/apt/sources.list.d
+
+#Run the cassandra cql to create the keyspaces.  Note this only works for the
+# us-east region for the replication factor on the keyspaces
+
+
+#Install cassandra so we have the cli
+curl -L http://debian.datastax.com/debian/repo_key | apt-key add -
+
+sudo cat >> /etc/apt/sources.list.d/cassandra.sources.list << EOF
+deb http://debian.datastax.com/community stable main
+EOF
+
+apt-get update
+apt-get -y --force-yes install libcap2 cassandra=1.2.19
+/etc/init.d/cassandra stop
+
+#Get the first instance of cassandra
+cd /usr/share/usergrid/scripts
+CASSHOST=$(groovy get_first_instance.groovy cassandra)
+
+#We have to wait for cass to actually start before we can run our CQL.   Sleep 5 seconds between retries
+while ! echo exit | nc ${CASSHOST} 9160; do sleep 5; done
+
+#WE have to remove our -1 from the end, since us-east and us-west dont support -1 in cassandra
+CASS_REGION=${EC2_REGION%-1}
+
+#Update the keyspace replication and run the cql
+sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/update_locks.cql
+
+/usr/bin/cassandra-cli -h ${CASSHOST} -f  /usr/share/usergrid/cql/update_locks.cql
+
+
+#Update the keyspace region and run the cql
+sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/update_usergrid.cql
+
+/usr/bin/cassandra-cli -h ${CASSHOST} -f  /usr/share/usergrid/cql/update_usergrid.cql
+
+
+#Update the keyspace region and run the cql
+sed -i.bak "s/KEYSPACE_REGION/${CASS_REGION}/g" /usr/share/usergrid/cql/update_usergrid_applications.cql
+
+/usr/bin/cassandra-cli -h ${CASSHOST} -f  /usr/share/usergrid/cql/update_usergrid_applications.cql
+
+
+popd

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/src/main/groovy/NodeRegistry.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/NodeRegistry.groovy b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
index 7389073..6291f93 100644
--- a/stack/awscluster/src/main/groovy/NodeRegistry.groovy
+++ b/stack/awscluster/src/main/groovy/NodeRegistry.groovy
@@ -55,8 +55,9 @@ class NodeRegistry {
      * @param defNodeType
      */
     def searchNode(def nodeType) {
+        //order by create time, if we have a conflict, then order by item name
         def selectResult = sdbClient.select(new SelectRequest((String) \
-            "select * from `${domain}` where itemName() is not null and nodetype = '${nodeType}'  order by itemName()"))
+            "select * from `${domain}` where itemName() is not null AND createtime is not null AND nodetype = '${nodeType}'  order by createtime"))
         def result = []
 
         for (item in selectResult.getItems()) {
@@ -96,11 +97,14 @@ class NodeRegistry {
         } else {
             println "Registering..."
             def stackAtt = new ReplaceableAttribute("nodetype", nodeType, true)
+            def nowString = Calendar.getInstance(TimeZone.getTimeZone('UTC')).format("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
+            def createTime = new ReplaceableAttribute("createtime", nowString, true)
             def attrs = new ArrayList()
             attrs.add(stackAtt)
+            attrs.add(createTime)
             def par = new PutAttributesRequest(domain, hostName, attrs)
             sdbClient.putAttributes(par);
-            println "Registraition done."
+            println "Registration done."
             return true;
         }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index 345b5a1..5b599ec 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -59,8 +59,8 @@
         "CassNumServers": {
             "Description": "Number of Cass servers to start.",
             "Type": "Number",
-            "Default": "3",
-            "MinValue": "1"
+            "Default": "6",
+            "MinValue": "3"
         },
         "CassInstanceType": {
             "Description": "Instance type for Cass servers",
@@ -88,8 +88,8 @@
         "ESNumServers": {
             "Description": "Number of ES servers to start.",
             "Type": "Number",
-            "Default": "3",
-            "MinValue": "1"
+            "Default": "6",
+            "MinValue": "3"
         },
         "ESInstanceType": {
             "Description": "Instance type for ES servers",
@@ -106,17 +106,17 @@
         "SuperUserEmail": {
             "Description": "Email for superuser user",
             "Type": "String",
-            "Default": "dmjohnson+super@apigee.com"
+            "Default": "tnine+super@apigee.com"
         },
         "TestAdminUserEmail": {
             "Description": "Email for test admin user",
             "Type": "String",
-            "Default": "dmjohnson+admin@apigee.com"
+            "Default": "tnine+admin@apigee.com"
         },
         "NotificationEmail": {
             "Description": "Email for notifications",
             "Type": "String",
-            "Default": "dmjohnson+super@apigee.com"
+            "Default": "tnine+super@apigee.com"
         },
         "GraphiteInstanceType": {
             "Description": "Instance type for Graphite server",

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index 4b4ded6..48af6a2 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -126,11 +126,9 @@ public class CpSetup implements Setup {
             cpProps.put("collections.keyspace.strategy.class", 
                     cass.getProperties().get("cassandra.keyspace.strategy"));
 
-            cpProps.put("collections.keyspace.strategy.options", "replication_factor:" +  
+            cpProps.put("collections.keyspace.strategy.options",
                     cass.getProperties().get("cassandra.keyspace.replication"));
 
-            cpProps.put("cassandra.keyspace.strategy.options.replication_factor",
-                    cass.getProperties().get("cassandra.keyspace.replication"));
 
             logger.debug("Set Cassandra properties for Core Persistence: " + cpProps.toString() );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/54772ab0/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
index 1e3b1c1..2322b95 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/CassandraService.java
@@ -37,6 +37,7 @@ import org.apache.usergrid.persistence.IndexBucketLocator.IndexType;
 import org.apache.usergrid.persistence.cassandra.index.IndexBucketScanner;
 import org.apache.usergrid.persistence.cassandra.index.IndexScanner;
 import org.apache.usergrid.persistence.hector.CountingMutator;
+import org.apache.usergrid.utils.MapUtils;
 
 import me.prettyprint.cassandra.connection.HConnectionManager;
 import me.prettyprint.cassandra.model.ConfigurableConsistencyLevel;
@@ -309,6 +310,8 @@ public class CassandraService {
 
             //default read repair chance to 0.1
             cfDef.setReadRepairChance( 0.1d );
+            cfDef.setCompactionStrategy( "LeveledCompactionStrategy" );
+            cfDef.setCompactionStrategyOptions( new MapUtils.HashMapBuilder().map("sstable_size_in_mb", "512"  ) );
 
             cluster.addColumnFamily( cfDef, true );
             logger.info( "Created column family {} in keyspace {}", cfDef.getName(), keyspace );


[24/27] incubator-usergrid git commit: Fixes stale entity cleanup test to be predictable with sort order.

Posted by to...@apache.org.
Fixes stale entity cleanup test to be predictable with sort order.

Fixes de-index to pass source scope, rather than trying to determine it.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/75123da3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/75123da3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/75123da3

Branch: refs/heads/two-dot-o
Commit: 75123da3c7e53d82c363bb7f790798617023ec8b
Parents: 12b5028
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 12 12:04:22 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 12 12:04:22 2014 -0700

----------------------------------------------------------------------
 .../corepersistence/CpRelationManager.java      |  19 ++-
 .../results/FilteringLoader.java                |  26 +--
 .../results/ResultsLoaderFactory.java           |   9 +-
 .../results/ResultsLoaderFactoryImpl.java       |   6 +-
 .../batch/job/AbstractSchedulerRuntimeIT.java   |   2 +
 .../corepersistence/StaleIndexCleanupTest.java  | 163 ++++++++++---------
 .../index/impl/EsEntityIndexBatchImpl.java      |  12 +-
 .../impl/EntityConnectionIndexImplTest.java     | 129 +++++++++++++--
 .../org/apache/usergrid/rest/RestITSuite.java   |   2 +-
 .../org/apache/usergrid/ServiceITSuite.java     |   2 +-
 10 files changed, 235 insertions(+), 135 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 9a729b8..f7546b8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -921,6 +921,7 @@ public class CpRelationManager implements RelationManager {
 
         if ( query == null ) {
             query = new Query();
+            query.setCollection( collName );
         }
 
         headEntity = em.validate( headEntity );
@@ -968,11 +969,11 @@ public class CpRelationManager implements RelationManager {
 
             if ( results == null ) {
                 logger.debug( "Calling build results 1" );
-                results = buildResults( query, crs, collName );
+                results = buildResults( indexScope, query, crs, collName );
             }
             else {
                 logger.debug( "Calling build results 2" );
-                Results newResults = buildResults( query, crs, collName );
+                Results newResults = buildResults(indexScope,  query, crs, collName );
                 results.merge( newResults );
             }
 
@@ -1367,7 +1368,7 @@ public class CpRelationManager implements RelationManager {
             query = adjustQuery( query );
             CandidateResults crs = ei.search( indexScope, searchTypes,  query );
 
-            raw = buildResults( query, crs, query.getConnectionType() );
+            raw = buildResults( indexScope, query, crs, query.getConnectionType() );
         }
 
         if ( Level.ALL_PROPERTIES.equals( level ) ) {
@@ -1459,7 +1460,7 @@ public class CpRelationManager implements RelationManager {
         query = adjustQuery( query );
         CandidateResults crs = ei.search( indexScope, searchTypes, query );
 
-        return buildConnectionResults( query, crs, connection );
+        return buildConnectionResults( indexScope, query, crs, connection );
     }
 
 
@@ -1550,11 +1551,11 @@ public class CpRelationManager implements RelationManager {
     }
 
 
-    private Results buildConnectionResults(
-            Query query, CandidateResults crs, String connectionType ) {
+    private Results buildConnectionResults( final IndexScope indexScope,
+            final Query query, final CandidateResults crs, final String connectionType ) {
 
         if ( query.getLevel().equals( Level.ALL_PROPERTIES ) ) {
-            return buildResults( query, crs, connectionType );
+            return buildResults( indexScope, query, crs, connectionType );
         }
 
         final EntityRef sourceRef = new SimpleEntityRef( headEntity.getType(), headEntity.getUuid() );
@@ -1583,13 +1584,13 @@ public class CpRelationManager implements RelationManager {
      * @param crs Candidates to be considered for results
      * @param collName Name of collection or null if querying all types
      */
-    private Results buildResults( Query query, CandidateResults crs, String collName ) {
+    private Results buildResults( final IndexScope indexScope, final Query query, final CandidateResults crs, final String collName ) {
 
         logger.debug( "buildResults() for {} from {} candidates", collName, crs.size() );
 
         //get an instance of our results loader
         final ResultsLoader resultsLoader = this.resultsLoaderFactory.getLoader(
-                applicationScope, this.headEntity, query.getResultsLevel() );
+                applicationScope, indexScope, query.getResultsLevel() );
 
         //load the results
         final Results results = resultsLoader.loadResults( crs );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 3c6e96d..d6e63aa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -62,24 +62,23 @@ public class FilteringLoader implements ResultsLoader {
 
     private final CpManagerCache managerCache;
     private final ResultsVerifier resultsVerifier;
-    private final Id ownerId;
     private final ApplicationScope applicationScope;
+    private final IndexScope indexScope;
     private final EntityIndexBatch indexBatch;
 
 
     /**
      * Create an instance of a filter loader
      * @param managerCache The manager cache to load
-     * @param resultsVerifier
-     * @param ownerId
-     * @param applicationScope
+     * @param resultsVerifier  The verifier to verify the candidate results
+     * @param applicationScope The application scope to perform the load
+     * @param indexScope The index scope used in the search
      */
-    protected FilteringLoader( final CpManagerCache managerCache, final ResultsVerifier resultsVerifier,
-                               final EntityRef ownerId, final ApplicationScope applicationScope ) {
+    protected FilteringLoader( final CpManagerCache managerCache, final ResultsVerifier resultsVerifier,  final ApplicationScope applicationScope, final IndexScope indexScope ) {
         this.managerCache = managerCache;
         this.resultsVerifier = resultsVerifier;
-        this.ownerId = new SimpleId( ownerId.getUuid(), ownerId.getType() );
         this.applicationScope = applicationScope;
+        this.indexScope = indexScope;
 
         final EntityIndex index = managerCache.getEntityIndex( applicationScope );
 
@@ -155,7 +154,7 @@ public class FilteringLoader implements ResultsLoader {
 
                 //deindex this document, and remove the previous maxVersion
                 //we have to deindex this from our ownerId, since this is what gave us the reference
-                deIndex( indexBatch, ownerId, previousMax );
+                indexBatch.deindex( indexScope, previousMax );
                 groupedByScopes.remove( collectionType, previousMax );
 
 
@@ -212,7 +211,7 @@ public class FilteringLoader implements ResultsLoader {
 
                 //ask the loader if this is valid, if not discard it and de-index it
                 if ( !resultsVerifier.isValid( cr ) ) {
-                    deIndex( indexBatch, ownerId, cr );
+                    indexBatch.deindex( indexScope, cr );
                     continue;
                 }
 
@@ -236,13 +235,4 @@ public class FilteringLoader implements ResultsLoader {
     }
 
 
-    protected void deIndex( final EntityIndexBatch batch, final Id ownerId, 
-            final CandidateResult candidateResult ) {
-
-        IndexScope indexScope = new IndexScopeImpl( 
-            ownerId,
-            CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()) );
-
-        batch.deindex( indexScope, candidateResult );
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
index 779d604..14fbf88 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactory.java
@@ -22,6 +22,7 @@ package org.apache.usergrid.corepersistence.results;
 
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.query.Query;
 
 
@@ -31,8 +32,10 @@ import org.apache.usergrid.persistence.index.query.Query;
 public interface ResultsLoaderFactory {
 
     /**
-     * Get the load for results
+     * Get the loader for results
+     * @param applicationScope The application scope used to load results
+     * @param indexScope The index scope used in the search
+     * @param
      */
-    public ResultsLoader getLoader( final ApplicationScope applicationScope, 
-            final EntityRef ownerId, final Query.Level resultsLevel );
+    public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope indexScope, final Query.Level resultsLevel );
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
index 0f39fe3..96928fb 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/ResultsLoaderFactoryImpl.java
@@ -23,6 +23,7 @@ package org.apache.usergrid.corepersistence.results;
 import org.apache.usergrid.corepersistence.CpManagerCache;
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.query.Query;
 
 import com.google.inject.Inject;
@@ -43,8 +44,7 @@ public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
 
 
     @Override
-    public ResultsLoader getLoader( final ApplicationScope applicationScope, 
-            final EntityRef ownerId, final Query.Level resultsLevel ) {
+    public ResultsLoader getLoader( final ApplicationScope applicationScope, final IndexScope scope, final Query.Level resultsLevel ) {
 
         ResultsVerifier verifier;
 
@@ -58,6 +58,6 @@ public class ResultsLoaderFactoryImpl implements ResultsLoaderFactory {
             verifier = new EntityVerifier(Query.MAX_LIMIT);
         }
 
-        return new FilteringLoader( managerCache, verifier, ownerId, applicationScope );
+        return new FilteringLoader( managerCache, verifier, applicationScope, scope );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
index cc31d2d..abf54cb 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
@@ -30,11 +30,13 @@ import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.ClassRule;
+import org.junit.Ignore;
 
 
 /**
  * Class to test job runtimes
  */
+@Ignore("These tests no longer work with schared spring context. Need to re-evaluate")
 public class AbstractSchedulerRuntimeIT {
 
     protected static final int DEFAULT_COUNT = 10;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index bbd19ce..fa9f9df 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -15,19 +15,26 @@
  */
 package org.apache.usergrid.corepersistence;
 
-import com.fasterxml.uuid.UUIDComparator;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import org.apache.commons.lang3.RandomStringUtils;
+
 import org.apache.usergrid.AbstractCoreIT;
 import org.apache.usergrid.corepersistence.util.CpNamingUtils;
 import org.apache.usergrid.persistence.Entity;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.Results;
-import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
 import org.apache.usergrid.persistence.collection.CollectionScope;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
@@ -42,23 +49,20 @@ import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.model.entity.SimpleId;
-import org.junit.Assert;
+
+import com.fasterxml.uuid.UUIDComparator;
+
+import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 /**
  * Test on read style clean-up of stale ElasticSearch indexes.
  */
 public class StaleIndexCleanupTest extends AbstractCoreIT {
-    private static final Logger logger = LoggerFactory.getLogger(StaleIndexCleanupTest.class );
+    private static final Logger logger = LoggerFactory.getLogger( StaleIndexCleanupTest.class );
 
-    // take it easy on embedded Cassandra
-    private static final long writeDelayMs = 50;
-    private static final long readDelayMs = 50;
 
 
     /**
@@ -69,40 +73,38 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
 
         final EntityManager em = app.getEntityManager();
 
-        Entity thing = em.create("thing", new HashMap<String, Object>() {{
-            put("name", "thing1");
-        }});
+        Entity thing = em.create( "thing", new HashMap<String, Object>() {{
+            put( "name", "thing1" );
+        }} );
         em.refreshIndex();
-        
-        assertEquals( 1, queryCollectionCp("things", "thing", "select *").size() );
 
-        org.apache.usergrid.persistence.model.entity.Entity cpEntity = getCpEntity(thing);
+        assertEquals( 1, queryCollectionCp( "things", "thing", "select *" ).size() );
+
+        org.apache.usergrid.persistence.model.entity.Entity cpEntity = getCpEntity( thing );
         UUID oldVersion = cpEntity.getVersion();
 
-        em.updateProperties(thing, new HashMap<String, Object>() {{
-            put("stuff", "widget");
-        }});
+        em.updateProperties( thing, new HashMap<String, Object>() {{
+            put( "stuff", "widget" );
+        }} );
         em.refreshIndex();
 
-        org.apache.usergrid.persistence.model.entity.Entity cpUpdated = getCpEntity(thing);
-        assertEquals( "widget", cpUpdated.getField("stuff").getValue());
+        org.apache.usergrid.persistence.model.entity.Entity cpUpdated = getCpEntity( thing );
+        assertEquals( "widget", cpUpdated.getField( "stuff" ).getValue() );
         UUID newVersion = cpUpdated.getVersion();
 
-        assertTrue( "New version is greater than old", 
-                UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
+        assertTrue( "New version is greater than old", UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
 
-        assertEquals( 2, queryCollectionCp("things", "thing", "select *").size() );
+        assertEquals( 2, queryCollectionCp( "things", "thing", "select *" ).size() );
     }
 
 
     /**
-     * Test that the CpRelationManager cleans up and stale indexes that it finds when it is 
-     * building search results.
+     * Test that the CpRelationManager cleans up and stale indexes that it finds when it is building search results.
      */
     @Test
     public void testStaleIndexCleanup() throws Exception {
 
-        logger.info("Started testStaleIndexCleanup()");
+        logger.info( "Started testStaleIndexCleanup()" );
 
         // TODO: turn off post processing stuff that cleans up stale entities 
 
@@ -111,139 +113,142 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         final int numEntities = 10;
         final int numUpdates = 3;
 
+        final AtomicInteger updateCount =  new AtomicInteger(  );
+
         // create lots of entities
-        final List<Entity> things = new ArrayList<Entity>(numEntities);
-        for ( int i=0; i<numEntities; i++) {
+        final List<Entity> things = new ArrayList<Entity>( numEntities );
+        for ( int i = 0; i < numEntities; i++ ) {
             final String thingName = "thing" + i;
-            things.add( em.create("thing", new HashMap<String, Object>() {{
-                put("name", thingName);
-            }}));
-            Thread.sleep( writeDelayMs );
+            things.add( em.create( "thing", new HashMap<String, Object>() {{
+                put( "name", thingName );
+                put( "updateCount", updateCount.getAndIncrement() );
+            }} ) );
+
         }
         em.refreshIndex();
 
-        CandidateResults crs = queryCollectionCp( "things", "thing", "select *");
+        CandidateResults crs = queryCollectionCp( "things", "thing", "select * order by updateCount asc" );
         Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
 
         // update each one a bunch of times
         int count = 0;
 
-        List<Entity> maxVersions = new ArrayList<>(numEntities);
+        List<Entity> maxVersions = new ArrayList<>( numEntities );
+
+
+
 
         for ( Entity thing : things ) {
 
             Entity toUpdate = null;
 
-            for ( int j=0; j<numUpdates; j++) {
+            for ( int j = 0; j < numUpdates; j++ ) {
 
                 toUpdate = em.get( thing.getUuid() );
-                toUpdate.setProperty( "property"  + j, RandomStringUtils.randomAlphanumeric(10));
-                em.update(toUpdate);
+                //update the update count, so we'll order from the first entity created to the last
+                toUpdate.setProperty( "updateCount", updateCount.getAndIncrement() );
+                em.update( toUpdate );
 
-                Thread.sleep( writeDelayMs );
 
                 count++;
                 if ( count % 100 == 0 ) {
-                    logger.info("Updated {} of {} times", count, numEntities * numUpdates);
+                    logger.info( "Updated {} of {} times", count, numEntities * numUpdates );
                 }
             }
 
             maxVersions.add( toUpdate );
         }
+
         em.refreshIndex();
 
         // query Core Persistence directly for total number of result candidates
-        crs = queryCollectionCp("things", "thing", "select *");
-        Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
+        crs = queryCollectionCp( "things", "thing", "select * order by updateCount asc" );
+        Assert.assertEquals( "Expect stale candidates", numEntities * ( numUpdates + 1 ), crs.size() );
 
         // query EntityManager for results and page through them
         // should return numEntities becuase it filters out the stale entities
-        final int limit  = 8;
-        Query q = Query.fromQL("select *");
+        final int limit = 8;
+
+        //we order by updateCount asc, this forces old versions to appear first, otherwise, we don't clean them up in
+        // our versions
+        Query q = Query.fromQL( "select * order by updateCount asc" );
         q.setLimit( limit );
         int thingCount = 0;
         String cursor = null;
 
         int index = 0;
 
+
         do {
-            Results results = em.searchCollection( em.getApplicationRef(), "things", q);
+            Results results = em.searchCollection( em.getApplicationRef(), "things", q );
             thingCount += results.size();
 
-            logger.debug("Retrieved total of {} entities", thingCount );
+            logger.debug( "Retrieved total of {} entities", thingCount );
 
             cursor = results.getCursor();
             if ( cursor != null && thingCount < numEntities ) {
                 assertEquals( limit, results.size() );
             }
 
-            for (int i = 0; i < results.size(); i ++, index++){
+            for ( int i = 0; i < results.size(); i++, index++ ) {
 
-                final Entity returned = results.getEntities().get( i);
+                final Entity returned = results.getEntities().get( i );
 
                 //last entities appear first
                 final Entity expected = maxVersions.get( index );
-                assertEquals("correct entity returned", expected, returned);
-
+                assertEquals( "correct entity returned", expected, returned );
             }
-
-        } while ( cursor != null );
+        }
+        while ( cursor != null );
 
         assertEquals( "Expect no stale candidates", numEntities, thingCount );
 
+
         em.refreshIndex();
 
-        // EntityManager should have kicked off a batch cleanup of those stale indexes
-        // wait a second for batch cleanup to complete
-        Thread.sleep(600);
 
         // query for total number of result candidates = numEntities
-        crs = queryCollectionCp("things", "thing", "select *");
-        Assert.assertEquals( "Expect stale candidates de-indexed", numEntities, crs.size());
+        crs = queryCollectionCp( "things", "thing", "select *" );
+        Assert.assertEquals( "Expect stale candidates de-indexed", numEntities, crs.size() );
     }
 
 
-    /** 
+    /**
      * Go around EntityManager and get directly from Core Persistence.
      */
     private org.apache.usergrid.persistence.model.entity.Entity getCpEntity( EntityRef eref ) {
 
         EntityManager em = app.getEntityManager();
 
-        CollectionScope cs = new CollectionScopeImpl(
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ));
+        CollectionScope cs = new CollectionScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
+                new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
+                CpNamingUtils.getCollectionScopeNameFromEntityType( eref.getType() ) );
 
-        EntityCollectionManagerFactory ecmf = 
-                CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
+        EntityCollectionManagerFactory ecmf = CpSetup.getInjector().getInstance( EntityCollectionManagerFactory.class );
 
-        EntityCollectionManager ecm = ecmf.createCollectionManager(cs);
+        EntityCollectionManager ecm = ecmf.createCollectionManager( cs );
 
-        return ecm.load( new SimpleId( eref.getUuid(), eref.getType()))
-                .toBlocking().lastOrDefault(null);
-    } 
+        return ecm.load( new SimpleId( eref.getUuid(), eref.getType() ) ).toBlocking().lastOrDefault( null );
+    }
 
 
-    /** 
-     * Go around EntityManager and execute query directly against Core Persistence.
-     * Results may include stale index entries.
+    /**
+     * Go around EntityManager and execute query directly against Core Persistence. Results may include stale index
+     * entries.
      */
-    private CandidateResults queryCollectionCp(final  String collName, final String type, final String query ) {
+    private CandidateResults queryCollectionCp( final String collName, final String type, final String query ) {
 
         EntityManager em = app.getEntityManager();
 
         EntityIndexFactory eif = CpSetup.getInjector().getInstance( EntityIndexFactory.class );
 
-        ApplicationScope as = new ApplicationScopeImpl( 
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION));
+        ApplicationScope as = new ApplicationScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ) );
         EntityIndex ei = eif.createEntityIndex( as );
 
-        IndexScope is = new IndexScopeImpl(
-            new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
-        Query rcq = Query.fromQL(query);
-        rcq.setLimit(10000); // no paging
+        IndexScope is = new IndexScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ),
+                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
+        Query rcq = Query.fromQL( query );
+        rcq.setLimit( 10000 ); // no paging
 
         return ei.search( is, SearchTypes.fromTypes( type ), rcq );
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index e6920f5..93ec9e3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -58,6 +58,7 @@ import org.apache.usergrid.persistence.model.field.UUIDField;
 import org.apache.usergrid.persistence.model.field.value.EntityObject;
 
 import com.google.common.base.Joiner;
+import com.sun.xml.internal.ws.developer.MemberSubmissionAddressing;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
@@ -135,7 +136,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         String indexId = createIndexDocId( entity, context );
 
-        log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
+        log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
 
         bulkRequest.add( client.prepareIndex( indexName, entityType, indexId ).setSource( entityAsMap ) );
 
@@ -149,20 +150,25 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
     public EntityIndexBatch deindex( final IndexScope indexScope, final Id id, final UUID version ) {
 
         IndexValidationUtils.validateIndexScope( indexScope );
+        ValidationUtils.verifyIdentity( id );
+        ValidationUtils.verifyVersion( version );
 
         final String context = createContextName( indexScope );
         final String entityType = id.getType();
 
+        String indexId = createIndexDocId( id, version, context );
+
 
         if ( log.isDebugEnabled() ) {
-            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   " + "name {} context{}, type {}",
+            log.debug( "De-indexing entity {}:{} in scope\n   app {}\n   owner {}\n   " + "name {} context{}, type {},",
                     new Object[] {
                             id.getType(), id.getUuid(), applicationScope.getApplication(), indexScope.getOwner(),
                             indexScope.getName(), context, entityType
                     } );
         }
 
-        String indexId = createIndexDocId( id, version, context );
+
+        log.debug( "De-indexing type {} with documentId '{}'" , entityType, indexId);
 
         bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 20dcbde..18c3494 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -57,8 +57,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
 
     private static final Logger log = LoggerFactory.getLogger( EntityConnectionIndexImplTest.class );
 
-//    @ClassRule
-//    public static ElasticSearchResource es = new ElasticSearchResource();
+    //    @ClassRule
+    //    public static ElasticSearchResource es = new ElasticSearchResource();
 
 
     @Inject
@@ -101,7 +101,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
 
 
         // create a person who likes muffins
-        Id personId =   new SimpleId( UUIDGenerator.newTimeUUID(), "person" ) ;
+        Id personId = new SimpleId( UUIDGenerator.newTimeUUID(), "person" );
 
 
         assertNotNull( personId );
@@ -113,7 +113,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
 
         //create another scope we index in, want to be sure these scopes are filtered
-        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ), "likes" );
+        IndexScope otherIndexScope =
+                new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ), "likes" );
 
         EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
 
@@ -164,42 +165,134 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         assertContains( muffin.getId(), likes );
         assertContains( oj.getId(), likes );
 
-          //now lets search for all explicitly
-        likes = personLikesIndex.search( searchScope,
-                SearchTypes.allTypes(),
-                Query.fromQL( "select *" ) );
+        //now lets search for all explicitly
+        likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select *" ) );
         assertEquals( 3, likes.size() );
         assertContains( egg.getId(), likes );
-        assertContains( muffin.getId(), likes);
+        assertContains( muffin.getId(), likes );
         assertContains( oj.getId(), likes );
 
 
-
         //now search all entity types with a query that returns a subset
         likes = personLikesIndex.search( searchScope,
                 SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
                 Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertContains( egg.getId(), likes);
+        assertContains( egg.getId(), likes );
         assertContains( muffin.getId(), likes );
 
 
-
         //now search with no types, we should get only the results that match
-        likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where stars = 5" ) );
+        likes = personLikesIndex
+                .search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertContains( egg.getId(), likes);
+        assertContains( egg.getId(), likes );
+        assertContains( muffin.getId(), likes );
+    }
+
+
+    @Test
+    public void testDelete() throws IOException {
+
+        Id appId = new SimpleId( "application" );
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        // create a muffin
+        Entity muffin = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "muffin" ) );
+
+        muffin = EntityIndexMapUtils.fromMap( muffin, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "flavor", "Blueberry" );
+            put( "stars", 5 );
+        }} );
+        EntityUtils.setVersion( muffin, UUIDGenerator.newTimeUUID() );
+
+        Entity egg = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "egg" ) );
+
+        egg = EntityIndexMapUtils.fromMap( egg, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "type", "scramble" );
+            put( "stars", 5 );
+        }} );
+        EntityUtils.setVersion( egg, UUIDGenerator.newTimeUUID() );
+
+        Entity oj = new Entity( new SimpleId( UUIDGenerator.newTimeUUID(), "juice" ) );
+
+        oj = EntityIndexMapUtils.fromMap( oj, new HashMap<String, Object>() {{
+            put( "size", "Large" );
+            put( "type", "pulpy" );
+            put( "stars", 3 );
+        }} );
+        EntityUtils.setVersion( oj, UUIDGenerator.newTimeUUID() );
+
+
+        // create a person who likes muffins
+        Id personId = new SimpleId( UUIDGenerator.newTimeUUID(), "person" );
+
+
+        assertNotNull( personId );
+        assertNotNull( personId.getType() );
+        assertNotNull( personId.getUuid() );
+
+        // index connection of "person Dave likes Large Blueberry muffin"
+
+        IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
+
+        //create another scope we index in, want to be sure these scopes are filtered
+        IndexScope otherIndexScope =
+                new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ), "likes" );
+
+        EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
+
+        EntityIndexBatch batch = personLikesIndex.createBatch();
+
+        //add to both scopes
+
+        //add a muffin
+        batch.index( searchScope, muffin );
+        batch.index( otherIndexScope, muffin );
+
+        //add the eggs
+        batch.index( searchScope, egg );
+        batch.index( otherIndexScope, egg );
+
+        //add the oj
+        batch.index( searchScope, oj );
+        batch.index( otherIndexScope, oj );
+
+        batch.executeAndRefresh();
+        personLikesIndex.refresh();
+
+        // now, let's search for muffins
+        CandidateResults likes = personLikesIndex.search( searchScope,
+                SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
+                Query.fromQL( "select *" ) );
+        assertEquals( 3, likes.size() );
+        assertContains( egg.getId(), likes );
         assertContains( muffin.getId(), likes );
+        assertContains( oj.getId(), likes );
+
+
+        //now delete them
+        batch.deindex( searchScope, egg );
+        batch.deindex( searchScope, muffin );
+        batch.deindex( searchScope, oj );
+        batch.executeAndRefresh();
+
+        likes = personLikesIndex.search( searchScope,
+                SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
+                Query.fromQL( "select *" ) );
+        assertEquals( 0, likes.size() );
     }
 
 
-    private void assertContains(final Id id, final CandidateResults results ){
-        for( CandidateResult result: results){
-            if(result.getId().equals(id)){
+    private void assertContains( final Id id, final CandidateResults results ) {
+        for ( CandidateResult result : results ) {
+            if ( result.getId().equals( id ) ) {
                 return;
             }
         }
 
-        fail(String.format("Could not find id %s in candidate results", id));
+        fail( String.format( "Could not find id %s in candidate results", id ) );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
index 788a4b3..3c737a6 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/RestITSuite.java
@@ -57,7 +57,7 @@ import org.apache.usergrid.rest.management.organizations.AdminEmailEncodingIT;
 public class RestITSuite {
 
     @ClassRule
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 
     @ClassRule
     public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/75123da3/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
index d4e9751..5edc4579 100644
--- a/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
+++ b/stack/services/src/test/java/org/apache/usergrid/ServiceITSuite.java
@@ -60,5 +60,5 @@ public class ServiceITSuite {
     public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts();
     
     @ClassRule
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 }


[08/27] incubator-usergrid git commit: Fixes missing zones. Just repeats zones for regions that don't have 4 zones. Done because we ran out of zones with c3.4xlarge with only 2 zones in east.

Posted by to...@apache.org.
Fixes missing zones.  Just repeats zones for regions that don't have 4 zones.  Done because we ran out of zones with c3.4xlarge with only 2 zones in east.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3495001a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3495001a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3495001a

Branch: refs/heads/two-dot-o
Commit: 3495001a80b1c5b286630be1409a87dc176efca7
Parents: eef2b70
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 7 13:49:22 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 7 13:49:22 2014 -0700

----------------------------------------------------------------------
 stack/awscluster/ugcluster-cf.json | 232 ++++++++++++++++++++++++--------
 1 file changed, 173 insertions(+), 59 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3495001a/stack/awscluster/ugcluster-cf.json
----------------------------------------------------------------------
diff --git a/stack/awscluster/ugcluster-cf.json b/stack/awscluster/ugcluster-cf.json
index c432923..345b5a1 100644
--- a/stack/awscluster/ugcluster-cf.json
+++ b/stack/awscluster/ugcluster-cf.json
@@ -212,18 +212,24 @@
                 "64": "ami-194a0429"
             }
         },
-        "TwoAZs": {
+        "FourAZs": {
             "ap-southeast-2": {
                 "AZ1": "ap-southeast-2a",
-                "AZ2": "ap-southeast-2b"
+                "AZ2": "ap-southeast-2b",
+                "AZ3": "ap-southeast-2a",
+                "AZ4": "ap-southeast-2b"
             },
             "us-east-1": {
                 "AZ1": "us-east-1b",
-                "AZ2": "us-east-1c"
+                "AZ2": "us-east-1c",
+                "AZ3": "us-east-1d",
+                "AZ4": "us-east-1e"
             },
             "us-west-2": {
                 "AZ1": "us-west-2a",
-                "AZ2": "us-west-2b"
+                "AZ2": "us-west-2b",
+                "AZ3": "us-west-2c",
+                "AZ4": "us-west-2a"
             }
         }
     },
@@ -375,7 +381,7 @@
                 "AvailabilityZones": [
                     {
                         "Fn::FindInMap": [
-                            "TwoAZs",
+                            "FourAZs",
                             {
                                 "Ref": "AWS::Region"
                             },
@@ -384,13 +390,31 @@
                     },
                     {
                         "Fn::FindInMap": [
-                            "TwoAZs",
+                            "FourAZs",
                             {
                                 "Ref": "AWS::Region"
                             },
                             "AZ2"
                         ]
-                    }
+                    },
+                    {
+                         "Fn::FindInMap": [
+                             "FourAZs",
+                             {
+                                 "Ref": "AWS::Region"
+                             },
+                             "AZ3"
+                         ]
+                     },
+                    {
+                         "Fn::FindInMap": [
+                             "FourAZs",
+                             {
+                                 "Ref": "AWS::Region"
+                             },
+                             "AZ4"
+                         ]
+                     }
                 ],
                 "LaunchConfigurationName": {
                     "Ref": "GraphiteAutoScalingLaunchConfiguration"
@@ -573,10 +597,10 @@
                         "Type": "AWS::AutoScaling::AutoScalingGroup",
                         "Version": "2014-07-24",
                         "Properties": {
-                            "AvailabilityZones": [
+                          "AvailabilityZones": [
                                 {
                                     "Fn::FindInMap": [
-                                        "TwoAZs",
+                                        "FourAZs",
                                         {
                                             "Ref": "AWS::Region"
                                         },
@@ -585,13 +609,31 @@
                                 },
                                 {
                                     "Fn::FindInMap": [
-                                        "TwoAZs",
+                                        "FourAZs",
                                         {
                                             "Ref": "AWS::Region"
                                         },
                                         "AZ2"
                                     ]
-                                }
+                                },
+                                {
+                                     "Fn::FindInMap": [
+                                         "FourAZs",
+                                         {
+                                             "Ref": "AWS::Region"
+                                         },
+                                         "AZ3"
+                                     ]
+                                 },
+                                {
+                                     "Fn::FindInMap": [
+                                         "FourAZs",
+                                         {
+                                             "Ref": "AWS::Region"
+                                         },
+                                         "AZ4"
+                                     ]
+                                 }
                             ],
                             "LaunchConfigurationName": {
                                 "Ref": "OpsCenterAutoScalingLaunchConfiguration"
@@ -827,10 +869,10 @@
             "Type": "AWS::AutoScaling::AutoScalingGroup",
             "Version": "2009-05-15",
             "Properties": {
-                "AvailabilityZones": [
+              "AvailabilityZones": [
                     {
                         "Fn::FindInMap": [
-                            "TwoAZs",
+                            "FourAZs",
                             {
                                 "Ref": "AWS::Region"
                             },
@@ -839,13 +881,31 @@
                     },
                     {
                         "Fn::FindInMap": [
-                            "TwoAZs",
+                            "FourAZs",
                             {
                                 "Ref": "AWS::Region"
                             },
                             "AZ2"
                         ]
-                    }
+                    },
+                    {
+                         "Fn::FindInMap": [
+                             "FourAZs",
+                             {
+                                 "Ref": "AWS::Region"
+                             },
+                             "AZ3"
+                         ]
+                     },
+                    {
+                         "Fn::FindInMap": [
+                             "FourAZs",
+                             {
+                                 "Ref": "AWS::Region"
+                             },
+                             "AZ4"
+                         ]
+                     }
                 ],
                 "LaunchConfigurationName": {
                     "Ref": "CassAutoScalingLaunchConfiguration"
@@ -988,26 +1048,44 @@
             "Type": "AWS::AutoScaling::AutoScalingGroup",
             "Version": "2009-05-15",
             "Properties": {
-                "AvailabilityZones": [
-                    {
-                        "Fn::FindInMap": [
-                            "TwoAZs",
-                            {
-                                "Ref": "AWS::Region"
-                            },
-                            "AZ1"
-                        ]
-                    },
-                    {
-                        "Fn::FindInMap": [
-                            "TwoAZs",
-                            {
-                                "Ref": "AWS::Region"
-                            },
-                            "AZ2"
-                        ]
-                    }
-                ],
+              "AvailabilityZones": [
+                      {
+                          "Fn::FindInMap": [
+                              "FourAZs",
+                              {
+                                  "Ref": "AWS::Region"
+                              },
+                              "AZ1"
+                          ]
+                      },
+                      {
+                          "Fn::FindInMap": [
+                              "FourAZs",
+                              {
+                                  "Ref": "AWS::Region"
+                              },
+                              "AZ2"
+                          ]
+                      },
+                      {
+                           "Fn::FindInMap": [
+                               "FourAZs",
+                               {
+                                   "Ref": "AWS::Region"
+                               },
+                               "AZ3"
+                           ]
+                       },
+                      {
+                           "Fn::FindInMap": [
+                               "FourAZs",
+                               {
+                                   "Ref": "AWS::Region"
+                               },
+                               "AZ4"
+                           ]
+                       }
+                  ],
                 "LaunchConfigurationName": {
                     "Ref": "ESAutoScalingLaunchConfiguration"
                 },
@@ -1151,26 +1229,44 @@
             "Type": "AWS::AutoScaling::AutoScalingGroup",
             "Version": "2009-05-15",
             "Properties": {
-                "AvailabilityZones": [
-                    {
-                        "Fn::FindInMap": [
-                            "TwoAZs",
-                            {
-                                "Ref": "AWS::Region"
-                            },
-                            "AZ1"
-                        ]
-                    },
-                    {
-                        "Fn::FindInMap": [
-                            "TwoAZs",
-                            {
-                                "Ref": "AWS::Region"
-                            },
-                            "AZ2"
-                        ]
-                    }
-                ],
+              "AvailabilityZones": [
+                      {
+                          "Fn::FindInMap": [
+                              "FourAZs",
+                              {
+                                  "Ref": "AWS::Region"
+                              },
+                              "AZ1"
+                          ]
+                      },
+                      {
+                          "Fn::FindInMap": [
+                              "FourAZs",
+                              {
+                                  "Ref": "AWS::Region"
+                              },
+                              "AZ2"
+                          ]
+                      },
+                      {
+                           "Fn::FindInMap": [
+                               "FourAZs",
+                               {
+                                   "Ref": "AWS::Region"
+                               },
+                               "AZ3"
+                           ]
+                       },
+                      {
+                           "Fn::FindInMap": [
+                               "FourAZs",
+                               {
+                                   "Ref": "AWS::Region"
+                               },
+                               "AZ4"
+                           ]
+                       }
+                  ],
                 "LaunchConfigurationName": {
                     "Ref": "RestAutoScalingLaunchConfiguration"
                 },
@@ -1277,10 +1373,10 @@
         "RestElasticLoadBalancer": {
             "Type": "AWS::ElasticLoadBalancing::LoadBalancer",
             "Properties": {
-                "AvailabilityZones": [
+              "AvailabilityZones": [
                     {
                         "Fn::FindInMap": [
-                            "TwoAZs",
+                            "FourAZs",
                             {
                                 "Ref": "AWS::Region"
                             },
@@ -1289,13 +1385,31 @@
                     },
                     {
                         "Fn::FindInMap": [
-                            "TwoAZs",
+                            "FourAZs",
                             {
                                 "Ref": "AWS::Region"
                             },
                             "AZ2"
                         ]
-                    }
+                    },
+                    {
+                         "Fn::FindInMap": [
+                             "FourAZs",
+                             {
+                                 "Ref": "AWS::Region"
+                             },
+                             "AZ3"
+                         ]
+                     },
+                    {
+                         "Fn::FindInMap": [
+                             "FourAZs",
+                             {
+                                 "Ref": "AWS::Region"
+                             },
+                             "AZ4"
+                         ]
+                     }
                 ],
                 "Listeners": [
                     {


[17/27] incubator-usergrid git commit: Removed unused code and made cassandra resource completely external

Posted by to...@apache.org.
Removed unused code and made cassandra resource completely external


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/a103e7d7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/a103e7d7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/a103e7d7

Branch: refs/heads/two-dot-o
Commit: a103e7d713b718730479eb25143574150913dd3b
Parents: e16dd43
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 16:49:34 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 16:49:34 2014 -0700

----------------------------------------------------------------------
 .../usergrid/cassandra/CassandraResource.java   | 518 +++++--------------
 1 file changed, 134 insertions(+), 384 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/a103e7d7/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
index 401e026..3de0144 100644
--- a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
@@ -17,15 +17,8 @@
 package org.apache.usergrid.cassandra;
 
 
-import java.io.BufferedReader;
 import java.io.File;
-import java.io.FileWriter;
 import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
 import java.util.Properties;
 
 import org.junit.rules.ExternalResource;
@@ -33,30 +26,22 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
-import org.springframework.core.env.PropertySource;
-import org.yaml.snakeyaml.Yaml;
-import org.apache.cassandra.service.CassandraDaemon;
+
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.ArrayUtils;
 import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.math.RandomUtils;
 
+
 /**
  * A JUnit {@link org.junit.rules.ExternalResource} designed to start up Cassandra once in a TestSuite or test Class as
- * a shared external resource across test cases and shut it down after the TestSuite has completed.
- * <p/>
- * This external resource is completely isolated in terms of the files used and the ports selected if the {@link
- * AvailablePortFinder} is used with it.
- * <p/>
- * Note that for this resource to work properly, a project.properties file must be placed in the src/test/resources
- * directory with the following stanza in the project pom's build section:
- * <p/>
- * <testResources> <testResource> <directory>src/test/resources</directory> <filtering>true</filtering> <includes>
- * <include>**\/*.properties</include> <include>**\/*.xml</include> </includes> </testResource> </testResources>
- * <p/>
- * The following property expansion macro should be placed in this project.properties file:
- * <p/>
+ * a shared external resource across test cases and shut it down after the TestSuite has completed. <p/> This external
+ * resource is completely isolated in terms of the files used and the ports selected if the {@link AvailablePortFinder}
+ * is used with it. <p/> Note that for this resource to work properly, a project.properties file must be placed in the
+ * src/test/resources directory with the following stanza in the project pom's build section: <p/> <testResources>
+ * <testResource> <directory>src/test/resources</directory> <filtering>true</filtering> <includes>
+ * <include>**\/*.properties</include> <include>**\/*.xml</include> </includes> </testResource> </testResources> <p/>
+ * The following property expansion macro should be placed in this project.properties file: <p/>
  * target.directory=${pom.build.directory}
  *
  * TODO this class does 2 things.  It loads spring and starts cassandra.  We should separate these concerns.
@@ -93,34 +78,27 @@ public class CassandraResource extends ExternalResource {
     private int nativeTransportPort = DEFAULT_NATIVE_TRANSPORT_PORT;
 
     private ConfigurableApplicationContext applicationContext;
-    private CassandraDaemon cassandraDaemon;
     private SchemaManager schemaManager;
 
     private static CassandraResource instance;
     private Thread shutdown;
 
-    private Process process = null;
-
     private static Properties properties = null;
 
-    private boolean forkCassandra = false;
-    private boolean externalCassandra = false;
 
-     /**
-     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the 
-     * default SchemaManager for Cassandra.
+    /**
+     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the default SchemaManager for
+     * Cassandra.
      */
-    @SuppressWarnings("UnusedDeclaration")
+    @SuppressWarnings( "UnusedDeclaration" )
     CassandraResource() throws IOException {
-        this( null, DEFAULT_RPC_PORT, DEFAULT_STORAGE_PORT, DEFAULT_SSL_STORAGE_PORT, 
-                DEFAULT_NATIVE_TRANSPORT_PORT );
-
+        this( null, DEFAULT_RPC_PORT, DEFAULT_STORAGE_PORT, DEFAULT_SSL_STORAGE_PORT, DEFAULT_NATIVE_TRANSPORT_PORT );
     }
 
 
     /**
-     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the 
-     * specified SchemaManager for Cassandra.
+     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the specified SchemaManager for
+     * Cassandra.
      */
     CassandraResource( String schemaManagerName, int rpcPort, int storagePort, int sslStoragePort,
                        int nativeTransportPort ) {
@@ -143,27 +121,24 @@ public class CassandraResource extends ExternalResource {
 
         try {
             String[] locations = { "usergrid-properties-context.xml" };
-            ConfigurableApplicationContext appContext = 
-                    new ClassPathXmlApplicationContext( locations );
-            
-            Properties properties = (Properties)appContext.getBean("properties");
-            properties.putAll(ArrayUtils.toMap(this.getProjectProperties().entrySet().toArray(new Object[]{})));
-            String forkString = properties.getProperty("cassandra.startup");
-            forkCassandra = "forked".equals( forkString );
-            externalCassandra = "external".equals( forkString );
-
-        } catch (Exception ex) {
-            throw new RuntimeException("Error getting properties", ex);
+            ConfigurableApplicationContext appContext = new ClassPathXmlApplicationContext( locations );
+
+            Properties properties = ( Properties ) appContext.getBean( "properties" );
+            properties.putAll( ArrayUtils.toMap( this.getProjectProperties().entrySet().toArray( new Object[] { } ) ) );
+            String forkString = properties.getProperty( "cassandra.startup" );
+        }
+        catch ( Exception ex ) {
+            throw new RuntimeException( "Error getting properties", ex );
         }
-//        throw new RuntimeException("My debugging skills are terrible!");
     }
 
 
     /**
-     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the specified 
-     * SchemaManager for Cassandra.
+     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the specified SchemaManager for
+     * Cassandra.
      */
-    public CassandraResource( int rpcPort, int storagePort, int sslStoragePort, int nativeTransportPort ) throws IOException {
+    public CassandraResource( int rpcPort, int storagePort, int sslStoragePort, int nativeTransportPort )
+            throws IOException {
         this( null, rpcPort, storagePort, sslStoragePort, nativeTransportPort );
     }
 
@@ -208,7 +183,7 @@ public class CassandraResource extends ExternalResource {
      *
      * @return the temporary directory
      */
-    @SuppressWarnings("UnusedDeclaration")
+    @SuppressWarnings( "UnusedDeclaration" )
     public File getTemporaryDirectory() {
         return tempDir;
     }
@@ -254,7 +229,12 @@ public class CassandraResource extends ExternalResource {
 
     @Override
     public String toString() {
-        return "\n" + "cassandra.yaml = " + new File(tempDir, "cassandra.yaml") + "\n" + RPC_PORT_KEY + " = " + rpcPort + "\n" + STORAGE_PORT_KEY + " = " + storagePort + "\n" + SSL_STORAGE_PORT_KEY + " = " + sslStoragePort + "\n" + NATIVE_TRANSPORT_PORT_KEY + " = " + nativeTransportPort + "\n" + DATA_FILE_DIR_KEY + " = " + new File(tempDir, "data").toString() + "\n" + COMMIT_FILE_DIR_KEY + " = " + new File(tempDir, "commitlog").toString() + "\n" + SAVED_CACHES_DIR_KEY + " = " + new File(tempDir, "saved_caches").toString() + "\n";
+        return "\n" + "cassandra.yaml = " + new File( tempDir, "cassandra.yaml" ) + "\n" + RPC_PORT_KEY + " = "
+                + rpcPort + "\n" + STORAGE_PORT_KEY + " = " + storagePort + "\n" + SSL_STORAGE_PORT_KEY + " = "
+                + sslStoragePort + "\n" + NATIVE_TRANSPORT_PORT_KEY + " = " + nativeTransportPort + "\n"
+                + DATA_FILE_DIR_KEY + " = " + new File( tempDir, "data" ).toString() + "\n" + COMMIT_FILE_DIR_KEY
+                + " = " + new File( tempDir, "commitlog" ).toString() + "\n" + SAVED_CACHES_DIR_KEY + " = " + new File(
+                tempDir, "saved_caches" ).toString() + "\n";
     }
 
 
@@ -294,318 +274,84 @@ public class CassandraResource extends ExternalResource {
             if ( isReady() ) {
                 return;
             }
-            
-//            if ( forkCassandra ) {
-//                startCassandraForked();
-//            } else if (externalCassandra) {
-//              startCassandraExternal();
-//            }else {
-//
-//                startCassandraEmbedded();
-//            }
-        }
-    }
-    private void addShutdownHook(){
-        Runtime.getRuntime().addShutdownHook( new Thread() {
-            @Override
-            public void run() {
-                after();
-            }
-        } );
-
-    }
-    private void startCassandraEmbedded() throws Throwable {
-
-        LOG.info( "-------------------------------------------------------------------");
-        LOG.info( "Initializing Embedded Cassandra at {} ...", tempDir.toString() );
-        LOG.info( "-------------------------------------------------------------------");
-        // Create temp directory, setup to create new File configuration there
-        File newYamlFile = new File( tempDir, "cassandra.yaml" );
-        URL newYamlUrl = FileUtils.toURLs( new File[] { newYamlFile } )[0];
-
-        // Read the original yaml file, make changes, and dump to new position in tmpdir
-        Yaml yaml = new Yaml();
-        @SuppressWarnings("unchecked") Map<String, Object> map =
-                ( Map<String, Object> ) yaml.load( ClassLoader.getSystemResourceAsStream( "cassandra.yaml" ) );
-        map.put( RPC_PORT_KEY, getRpcPort() );
-        map.put( STORAGE_PORT_KEY, getStoragePort() );
-        map.put( SSL_STORAGE_PORT_KEY, getSslStoragePort() );
-        map.put( NATIVE_TRANSPORT_PORT_KEY, getNativeTransportPort() );
-        map.put( COMMIT_FILE_DIR_KEY, new File( tempDir, "commitlog" ).toString() );
-        map.put( DATA_FILE_DIR_KEY, new String[] { new File( tempDir, "data" ).toString() } );
-        map.put( SAVED_CACHES_DIR_KEY, new File( tempDir, "saved_caches" ).toString() );
-        FileWriter writer = new FileWriter( newYamlFile );
-        yaml.dump( map, writer );
-        writer.flush();
-        writer.close();
-
-        // Fire up Cassandra by setting configuration to point to new yaml file
-        System.setProperty( "cassandra.url", "localhost:" + Integer.toString( rpcPort ) );
-        System.setProperty( "cassandra-foreground", "true" );
-        System.setProperty( "log4j.defaultInitOverride", "true" );
-        System.setProperty( "log4j.configuration", "log4j.properties" );
-        System.setProperty( "cassandra.ring_delay_ms", "100" );
-        System.setProperty( "cassandra.config", newYamlUrl.toString() );
-        System.setProperty( "cassandra.tempName", tempDir.getName() );
-
-        
-        //while ( !AvailablePortFinder.available( rpcPort ) || rpcPort == 9042 ) {
-        // why previously has a or condition of rpc == 9042?
-        while ( !AvailablePortFinder.available( rpcPort ) ) {
-            rpcPort++;
-        }
-        
-        while ( !AvailablePortFinder.available( storagePort ) ) {
-            storagePort++;
-        }
-        
-        while ( !AvailablePortFinder.available( sslStoragePort ) ) {
-            sslStoragePort++;
-        }
-        
-        while ( !AvailablePortFinder.available( nativeTransportPort ) ) {
-            nativeTransportPort++;
-        }
 
-        System.setProperty( "cassandra." + RPC_PORT_KEY, Integer.toString( rpcPort ) );
-        System.setProperty( "cassandra." + STORAGE_PORT_KEY, Integer.toString( storagePort ) );
-        System.setProperty( "cassandra." + SSL_STORAGE_PORT_KEY, Integer.toString( sslStoragePort ) );
-        System.setProperty( "cassandra." + NATIVE_TRANSPORT_PORT_KEY, Integer.toString( nativeTransportPort ) );
+            startCassandraExternal();
 
-        LOG.info("before() test, setting system properties for ports : "
-                + "[rpc, storage, sslStorage, native] = [{}, {}, {}, {}]", 
-                new Object[] {rpcPort, storagePort, sslStoragePort, nativeTransportPort});
-        if ( !newYamlFile.exists() ) {
-            throw new RuntimeException( "Cannot find new Yaml file: " + newYamlFile );
+            //            if ( forkCassandra ) {
+            //                startCassandraForked();
+            //            } else if (externalCassandra) {
+            //
+            //            }else {
+            //
+            //                startCassandraEmbedded();
+            //            }
         }
-        
-        cassandraDaemon = new CassandraDaemon();
-        cassandraDaemon.activate();
-
-//        Runtime.getRuntime().addShutdownHook( new Thread() {
-//            @Override
-//            public void run() {
-//                after();
-//            }
-//        } );
-        addShutdownHook();
-        String[] locations = { "usergrid-test-context.xml" };
-        applicationContext = new ClassPathXmlApplicationContext( locations );
-
-        loadSchemaManager( schemaManagerName );
-        initialized = true;
-        LOG.info( "External Cassandra resource at {} is ready!", tempDir.toString() );
-        lock.notifyAll();
     }
 
 
-    private void startCassandraForked() throws Throwable {
-
-        LOG.info( "-------------------------------------------------------------------");
-        LOG.info( "Initializing Forked Cassandra at {} ...", tempDir.toString() );
-        LOG.info( "-------------------------------------------------------------------");
-        // Create temp directory, setup to create new File configuration there
-        File newYamlFile = new File( tempDir, "cassandra.yaml" );
-        URL newYamlUrl = FileUtils.toURLs( new File[] { newYamlFile } )[0];
-
-        // Read the original yaml file, make changes, and dump to new position in tmpdir
-        Yaml yaml = new Yaml();
-        @SuppressWarnings("unchecked") Map<String, Object> map =
-                ( Map<String, Object> ) yaml.load( ClassLoader.getSystemResourceAsStream( "cassandra.yaml" ) );
-        map.put( RPC_PORT_KEY, getRpcPort() );
-        map.put( STORAGE_PORT_KEY, getStoragePort() );
-        map.put( SSL_STORAGE_PORT_KEY, getSslStoragePort() );
-        map.put( NATIVE_TRANSPORT_PORT_KEY, getNativeTransportPort() );
-        map.put( COMMIT_FILE_DIR_KEY, new File( tempDir, "commitlog" ).toString() );
-        map.put( DATA_FILE_DIR_KEY, new String[] { new File( tempDir, "data" ).toString() } );
-        map.put( SAVED_CACHES_DIR_KEY, new File( tempDir, "saved_caches" ).toString() );
-        FileWriter writer = new FileWriter( newYamlFile );
-        yaml.dump( map, writer );
-        writer.flush();
-        writer.close();
-
-        // Fire up Cassandra by setting configuration to point to new yaml file
-        System.setProperty( "cassandra.url", "localhost:" + Integer.toString( rpcPort ) );
-        System.setProperty( "cassandra-foreground", "true" );
-        System.setProperty( "log4j.defaultInitOverride", "true" );
-        System.setProperty( "log4j.configuration", "log4j.properties" );
-        System.setProperty( "cassandra.ring_delay_ms", "100" );
-        System.setProperty( "cassandra.config", newYamlUrl.toString() );
-        System.setProperty( "cassandra.tempName", tempDir.getName() );
-
-        
-        //while ( !AvailablePortFinder.available( rpcPort ) || rpcPort == 9042 ) {
-        // why previously has a or condition of rpc == 9042?
-        while ( !AvailablePortFinder.available( rpcPort ) ) {
-            rpcPort++;
-        }
-        
-        while ( !AvailablePortFinder.available( storagePort ) ) {
-            storagePort++;
-        }
-        
-        while ( !AvailablePortFinder.available( sslStoragePort ) ) {
-            sslStoragePort++;
-        }
-        
-        while ( !AvailablePortFinder.available( nativeTransportPort ) ) {
-            nativeTransportPort++;
-        }
-
-        System.setProperty( "cassandra." + RPC_PORT_KEY, Integer.toString( rpcPort ) );
-        System.setProperty( "cassandra." + STORAGE_PORT_KEY, Integer.toString( storagePort ) );
-        System.setProperty( "cassandra." + SSL_STORAGE_PORT_KEY, Integer.toString( sslStoragePort ) );
-        System.setProperty( "cassandra." + NATIVE_TRANSPORT_PORT_KEY, Integer.toString( nativeTransportPort ) );
-
-        LOG.info("before() test, setting system properties for ports : "
-                + "[rpc, storage, sslStorage, native] = [{}, {}, {}, {}]", 
-                new Object[] {rpcPort, storagePort, sslStoragePort, nativeTransportPort});
-
-        if ( !newYamlFile.exists() ) {
-            throw new RuntimeException( "Cannot find new Yaml file: " + newYamlFile );
-        }
-        
-        String javaHome = (String)System.getenv("JAVA_HOME");
-
-        String maxMemory = "-Xmx1000m";
-
-        ProcessBuilder pb = new ProcessBuilder(javaHome + "/bin/java", 
-                getJammArgument(), maxMemory,
-                "org.apache.usergrid.cassandra.CassandraMain", 
-                newYamlUrl.toString(), tempDir.getName(), 
-                getTargetDir() + "/src/test/resources/log4j.properties",
-                ""+rpcPort, ""+storagePort, ""+sslStoragePort, ""+nativeTransportPort );
-
-        // ensure Cassandra gets same classpath we have, but with...
-        String classpath = System.getProperty("java.class.path");
-        List<String> path = new ArrayList<String>();
-
-        String parts[] = classpath.split( File.pathSeparator );
-        for ( String part : parts ) {
-            if ( part.endsWith("test-classes") ) {
-                continue;
-            }
-            path.add(part);
-        }
-        String newClasspath = StringUtils.join( path, File.pathSeparator );
-
-        Map<String, String> env = pb.environment();
-        StringBuilder sb = new StringBuilder();
-        sb.append( newClasspath );
-        env.put("CLASSPATH", sb.toString());
-
-        pb.redirectErrorStream(true);
-
-        process = pb.start();
-
-        // use thread to log Cassandra output
-        new Thread( new Runnable() {
+    private void addShutdownHook() {
+        Runtime.getRuntime().addShutdownHook( new Thread() {
             @Override
             public void run() {
-                BufferedReader br = new BufferedReader(new InputStreamReader(process.getInputStream()));
-                String line = null;
-                try {
-                    while ((line = br.readLine()) != null) {
-                        LOG.info(line);
-                    }
-
-                } catch (Exception ex) {
-                    LOG.error("Error reading from Cassandra process", ex);
-                    return;
-                } 
+                after();
             }
-        }).start();
-
-//        Runtime.getRuntime().addShutdownHook( new Thread() {
-//            @Override
-//            public void run() {
-//                after();
-//            }
-//        } );
-        addShutdownHook();
-        // give C* time to start
-        Thread.sleep(5000);
+        } );
+    }
 
-        String[] locations = { "usergrid-test-context.xml" };
-        applicationContext = new ClassPathXmlApplicationContext( locations );
 
-        loadSchemaManager( schemaManagerName );
-        initialized = true;
-        LOG.info( "External Cassandra resource at {} is ready!", tempDir.toString() );
-        lock.notifyAll();
-    }
     private void startCassandraExternal() throws Throwable {
-        LOG.info( "-------------------------------------------------------------------");
-        LOG.info( "Initializing External Cassandra");
-        LOG.info( "-------------------------------------------------------------------");
-        LOG.info("before() test, setting system properties for ports : "
-                + "[rpc, storage, sslStorage, native] = [{}, {}, {}, {}]", 
-                new Object[] {rpcPort, storagePort, sslStoragePort, nativeTransportPort});
-        Thread.sleep(5000);
+        LOG.info( "-------------------------------------------------------------------" );
+        LOG.info( "Initializing External Cassandra" );
+        LOG.info( "-------------------------------------------------------------------" );
+        LOG.info( "before() test, setting system properties for ports : "
+                        + "[rpc, storage, sslStorage, native] = [{}, {}, {}, {}]",
+                new Object[] { rpcPort, storagePort, sslStoragePort, nativeTransportPort } );
+        Thread.sleep( 5000 );
         String[] locations = { "usergrid-test-context.xml" };
         applicationContext = new ClassPathXmlApplicationContext( locations );
-//        PropertySource ps=new PropertySource<String>();
-//        applicationContext.getEnvironment().getPropertySources().addLast(ps);
         applicationContext.refresh();
         loadSchemaManager( schemaManagerName );
         initialized = true;
-        
+
         LOG.info( "External Cassandra resource at {} is ready!", tempDir.toString() );
         lock.notifyAll();
-      
     }
 
+
     /** Stops Cassandra after a TestSuite or test Class executes. */
     @Override
     protected synchronized void after() {
         super.after();
-        if ( process != null ) {
-            process.destroy();
-        }
-        else { 
-            shutdown = new Thread() {
-                @Override
-                public void run() {
-                    try {
-                        Thread.currentThread();
-            Thread.sleep( 100L );
-                    }
-                    catch ( InterruptedException ignored ) {}
-                    if(externalCassandra){
-                        LOG.info( "Cleaning up external Cassandra instance");
-                    }else{
-                        LOG.info( "Shutting down Cassandra instance in {}", tempDir.toString() );
-                    }
-
-                    if ( schemaManager != null ) {
-                        LOG.info( "Destroying schemaManager..." );
-                        try {
-                            schemaManager.destroy();
-                        }
-                        catch ( Exception e ) {
-                            LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
-                        }
 
-                        LOG.info( "SchemaManager destroyed..." );
-                    }
+        shutdown = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    Thread.currentThread();
+                    Thread.sleep( 100L );
+                }
+                catch ( InterruptedException ignored ) {
+                }
 
-                    applicationContext.stop();
-                    LOG.info( "ApplicationContext stopped..." );
 
+                if ( schemaManager != null ) {
+                    LOG.info( "Destroying schemaManager..." );
                     try {
-                        if ( !externalCassandra && cassandraDaemon != null ) {
-                            LOG.info( "Deactivating CassandraDaemon..." );
-                            cassandraDaemon.deactivate();
-                        }
+                        schemaManager.destroy();
                     }
-                    catch ( Exception ex ) {
-                        LOG.error("Error deactivating Cassandra", ex);
+                    catch ( Exception e ) {
+                        LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
                     }
+
+                    LOG.info( "SchemaManager destroyed..." );
                 }
-            };
 
-            shutdown.start();
-        }
+                applicationContext.stop();
+                LOG.info( "ApplicationContext stopped..." );
+            }
+        };
+
+        shutdown.start();
     }
 
 
@@ -670,58 +416,63 @@ public class CassandraResource extends ExternalResource {
                 sslStoragePort = AvailablePortFinder.getNextAvailable( sslStoragePort );
             }
 
-            instance = new CassandraResource( 
-                schemaManagerName, rpcPort, storagePort, sslStoragePort, nativeTransportPort );
+            instance = new CassandraResource( schemaManagerName, rpcPort, storagePort, sslStoragePort,
+                    nativeTransportPort );
 
-            LOG.info("Created a new instance of CassandraResource: {}", instance);
-            LOG.info("Cassandra using ports {} and {}", storagePort, sslStoragePort);
+            LOG.info( "Created a new instance of CassandraResource: {}", instance );
+            LOG.info( "Cassandra using ports {} and {}", storagePort, sslStoragePort );
             return instance;
         }
     }
+
+
     public static CassandraResource newWithMavenAllocatedPorts() {
-      synchronized ( lock ) {
-          Properties props = new Properties();
-          try {
-        props.load(ClassLoader.getSystemResourceAsStream( "project.properties" ));
-
-          } catch (IOException e) {
-        LOG.error("Unable to load project properties: {}", e.getLocalizedMessage());
-      }
-          int rpcPort = Integer.parseInt(props.getProperty("cassandra.rpcPort", Integer.toString(CassandraResource.DEFAULT_RPC_PORT)));
-          int storagePort = Integer.parseInt(props.getProperty("cassandra.storagePort", Integer.toString(CassandraResource.DEFAULT_STORAGE_PORT))) ;
-          int sslStoragePort = Integer.parseInt(props.getProperty("cassandra.sslPort", Integer.toString(CassandraResource.DEFAULT_SSL_STORAGE_PORT)));
-          int nativeTransportPort = Integer.parseInt(props.getProperty("cassandra.nativeTransportPort", Integer.toString(CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT)));
-          String host = props.getProperty("cassandra.host", DEFAULT_HOST);
-//          int rpcPort = CassandraResource.DEFAULT_RPC_PORT;
-//          int storagePort = CassandraResource.DEFAULT_STORAGE_PORT ;
-//          int sslStoragePort = CassandraResource.DEFAULT_SSL_STORAGE_PORT;
-//          int nativeTransportPort = CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT;
-
-          System.setProperty( "cassandra.url", host+":" + Integer.toString( rpcPort ) );
-          System.setProperty( "cassandra.cluster", props.getProperty("cassandra.cluster","Usergrid") );
-          System.setProperty( "cassandra-foreground", "true" );
-          System.setProperty( "log4j.defaultInitOverride", "true" );
-          System.setProperty( "log4j.configuration", "log4j.properties" );
-          System.setProperty( "cassandra.ring_delay_ms", "100" );
-          
-          System.setProperty( "cassandra." + RPC_PORT_KEY, Integer.toString( rpcPort ) );
-          System.setProperty( "cassandra." + STORAGE_PORT_KEY, Integer.toString( storagePort ) );
-          System.setProperty( "cassandra." + SSL_STORAGE_PORT_KEY, Integer.toString( sslStoragePort ) );
-          System.setProperty( "cassandra." + NATIVE_TRANSPORT_PORT_KEY, Integer.toString( nativeTransportPort ) );
-
-          LOG.info("project.properties loaded properties for ports : "
-                  + "[rpc, storage, sslStorage, native] = [{}, {}, {}, {}]", 
-                  new Object[] {rpcPort, storagePort, sslStoragePort, nativeTransportPort});
-
-
-          instance = new CassandraResource( 
-              null, rpcPort, storagePort, sslStoragePort, nativeTransportPort );
-  
-          LOG.info("Created a new instance of CassandraResource: {}", instance);
-          LOG.info("Cassandra using ports {} and {}", storagePort, sslStoragePort);
-          
-          return instance;
-      }
+        synchronized ( lock ) {
+            Properties props = new Properties();
+            try {
+                props.load( ClassLoader.getSystemResourceAsStream( "project.properties" ) );
+            }
+            catch ( IOException e ) {
+                LOG.error( "Unable to load project properties: {}", e.getLocalizedMessage() );
+            }
+            int rpcPort = Integer.parseInt(
+                    props.getProperty( "cassandra.rpcPort", Integer.toString( CassandraResource.DEFAULT_RPC_PORT ) ) );
+            int storagePort = Integer.parseInt( props.getProperty( "cassandra.storagePort",
+                    Integer.toString( CassandraResource.DEFAULT_STORAGE_PORT ) ) );
+            int sslStoragePort = Integer.parseInt( props.getProperty( "cassandra.sslPort",
+                    Integer.toString( CassandraResource.DEFAULT_SSL_STORAGE_PORT ) ) );
+            int nativeTransportPort = Integer.parseInt( props.getProperty( "cassandra.nativeTransportPort",
+                    Integer.toString( CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT ) ) );
+            String host = props.getProperty( "cassandra.host", DEFAULT_HOST );
+            //          int rpcPort = CassandraResource.DEFAULT_RPC_PORT;
+            //          int storagePort = CassandraResource.DEFAULT_STORAGE_PORT ;
+            //          int sslStoragePort = CassandraResource.DEFAULT_SSL_STORAGE_PORT;
+            //          int nativeTransportPort = CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT;
+
+            System.setProperty( "cassandra.url", host + ":" + Integer.toString( rpcPort ) );
+            System.setProperty( "cassandra.cluster", props.getProperty( "cassandra.cluster", "Usergrid" ) );
+            System.setProperty( "cassandra-foreground", "true" );
+            System.setProperty( "log4j.defaultInitOverride", "true" );
+            System.setProperty( "log4j.configuration", "log4j.properties" );
+            System.setProperty( "cassandra.ring_delay_ms", "100" );
+
+            System.setProperty( "cassandra." + RPC_PORT_KEY, Integer.toString( rpcPort ) );
+            System.setProperty( "cassandra." + STORAGE_PORT_KEY, Integer.toString( storagePort ) );
+            System.setProperty( "cassandra." + SSL_STORAGE_PORT_KEY, Integer.toString( sslStoragePort ) );
+            System.setProperty( "cassandra." + NATIVE_TRANSPORT_PORT_KEY, Integer.toString( nativeTransportPort ) );
+
+            LOG.info( "project.properties loaded properties for ports : "
+                            + "[rpc, storage, sslStorage, native] = [{}, {}, {}, {}]",
+                    new Object[] { rpcPort, storagePort, sslStoragePort, nativeTransportPort } );
+
+
+            instance = new CassandraResource( null, rpcPort, storagePort, sslStoragePort, nativeTransportPort );
+
+            LOG.info( "Created a new instance of CassandraResource: {}", instance );
+            LOG.info( "Cassandra using ports {} and {}", storagePort, sslStoragePort );
+
+            return instance;
+        }
     }
 
 
@@ -772,7 +523,7 @@ public class CassandraResource extends ExternalResource {
 
 
     public static String getJammArgument() throws IOException {
-        return ( String ) getProjectProperties().get( JAMM_PATH ); 
+        return ( String ) getProjectProperties().get( JAMM_PATH );
     }
 
 
@@ -780,8 +531,7 @@ public class CassandraResource extends ExternalResource {
         if ( properties == null ) {
             properties = new Properties();
             properties.load( ClassLoader.getSystemResourceAsStream( PROPERTIES_FILE ) );
-        } 
+        }
         return properties;
     }
-
 }


[15/27] incubator-usergrid git commit: Turned down defaults on rest threading

Posted by to...@apache.org.
Turned down defaults on rest threading


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/c31e1b33
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/c31e1b33
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/c31e1b33

Branch: refs/heads/two-dot-o
Commit: c31e1b336648c73a374ba6fe875b5cd8c97455c9
Parents: 1ea42f8
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 16:21:08 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 16:21:08 2014 -0700

----------------------------------------------------------------------
 .../src/main/dist/init_instance/init_rest_server.sh          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/c31e1b33/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
index 7b6571c..924fb49 100644
--- a/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
+++ b/stack/awscluster/src/main/dist/init_instance/init_rest_server.sh
@@ -63,8 +63,8 @@ cd /usr/share/usergrid/init_instance
 # set Tomcat memory and threads based on instance type
 # use about 70% of RAM for heap
 export NOFILE=150000
-export TOMCAT_CONNECTIONS=10000
-export ACCEPT_COUNT=1600
+#export TOMCAT_CONNECTIONS=10000
+export ACCEPT_COUNT=100
 export NR_OPEN=1048576
 export FILE_MAX=761773
 
@@ -102,7 +102,7 @@ case `(curl http://169.254.169.254/latest/meta-data/instance-type)` in
 'c3.xlarge' )
     # total of 7.5g
     export TOMCAT_RAM=4096m
-    export TOMCAT_THREADS=7000
+    export TOMCAT_THREADS=1000
 ;;
 'c3.2xlarge' )
     # total of 15g
@@ -117,7 +117,7 @@ esac
 
 
 sed -i.bak "s/Xmx128m/Xmx${TOMCAT_RAM} -Xms${TOMCAT_RAM} -Dlog4j\.configuration=file:\/usr\/share\/usergrid\/lib\/log4j\.properties/g" /etc/default/tomcat7
-sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount=\"${ACCEPT_COUNT}\" maxConnections=\"${TOMCAT_CONNECTIONS}\"/g" /var/lib/tomcat7/conf/server.xml
+sed -i.bak "s/<Connector/<Connector maxThreads=\"${TOMCAT_THREADS}\" acceptCount=\"${ACCEPT_COUNT}\" maxConnections=\"${TOMCAT_THREADS}\"/g" /var/lib/tomcat7/conf/server.xml
 
 
 #Append our java opts for secret key


[19/27] incubator-usergrid git commit: Fixes test setup. Tests still broken

Posted by to...@apache.org.
Fixes test setup.  Tests still broken


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8ae98f82
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8ae98f82
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8ae98f82

Branch: refs/heads/two-dot-o
Commit: 8ae98f82f87c2ee632de0de5436d7b29931b8294
Parents: 500a327
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 18:12:43 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 18:12:43 2014 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/persistence/GeoIT.java  | 25 +++++++-------------
 1 file changed, 8 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8ae98f82/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index e07eeda..7e4493c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -52,10 +52,10 @@ public class GeoIT extends AbstractCoreIT {
     public void testGeo() throws Exception {
         LOG.info( "GeoIT.testGeo" );
 
-        UUID applicationId = setup.createApplication( "testOrganization", "testGeo" );
-        assertNotNull( applicationId );
 
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+
+        EntityManager em =  app.getEntityManager();
         assertNotNull( em );
 
         // create user at a location  
@@ -212,10 +212,8 @@ public class GeoIT extends AbstractCoreIT {
     @Test
     public void testPointPaging() throws Exception {
 
-        UUID applicationId = setup.createApplication( "testOrganization", "testPointPaging" );
-        assertNotNull( applicationId );
 
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+        EntityManager em = app.getEntityManager();
         assertNotNull( em );
 
         // save objects in a diagonal line from -90 -180 to 90 180
@@ -276,10 +274,7 @@ public class GeoIT extends AbstractCoreIT {
     @Test
     public void testSamePointPaging() throws Exception {
 
-        UUID applicationId = setup.createApplication( "testOrganization", "testSamePointPaging" );
-        assertNotNull( applicationId );
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+        EntityManager em =  app.getEntityManager();
         assertNotNull( em );
 
         // save objects in a diagonal line from -90 -180 to 90 180
@@ -326,10 +321,9 @@ public class GeoIT extends AbstractCoreIT {
     @Test
     public void testDistanceByLimit() throws Exception {
 
-        UUID applicationId = setup.createApplication( "testOrganization", "testDistanceByLimit" );
-        assertNotNull( applicationId );
 
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+
+        EntityManager em =  app.getEntityManager();
         assertNotNull( em );
 
         // save objects in a diagonal line from -90 -180 to 90 180
@@ -445,10 +439,7 @@ public class GeoIT extends AbstractCoreIT {
     @Test
     public void testDenseSearch() throws Exception {
 
-        UUID applicationId = setup.createApplication( "testOrganization", "testDenseSearch" );
-        assertNotNull( applicationId );
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+        EntityManager em =  app.getEntityManager();
         assertNotNull( em );
 
         // save objects in a diagonal line from -90 -180 to 90 180


[03/27] incubator-usergrid git commit: Finished updating search by versions. Removed query use to be more declarative.

Posted by to...@apache.org.
Finished updating search by versions.  Removed query use to be more declarative.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1debf19b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1debf19b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1debf19b

Branch: refs/heads/two-dot-o
Commit: 1debf19b30510b3f9f4abe57d8cba88efaec9208
Parents: ef77a5a
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 6 22:02:49 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 6 22:02:49 2014 -0700

----------------------------------------------------------------------
 .../index/impl/EsEntityIndexBatchImpl.java      |  2 +-
 .../index/impl/EsEntityIndexImpl.java           | 69 ++++++++++++++------
 .../persistence/index/impl/EntityIndexTest.java |  9 ---
 3 files changed, 49 insertions(+), 31 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1debf19b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 01f22c0..e6920f5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -259,7 +259,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         //but the fieldname
         //we have to prefix because we use query equality to seek this later.  TODO see if we can make this more declarative
-        entityMap.put( STRING_PREFIX+ ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
+        entityMap.put( ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
 
         return entityMap;
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1debf19b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index d7955c4..cb56c7d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -232,28 +232,21 @@ public class EsEntityIndexImpl implements EntityIndex {
         final String context = IndexingUtils.createContextName( indexScope );
         final String[] entityTypes = searchTypes.getTypeNames();
 
-        QueryBuilder qb = query.createQueryBuilder(context);
+        QueryBuilder qb = query.createQueryBuilder( context );
 
 
         SearchResponse searchResponse;
 
         if ( query.getCursor() == null ) {
             SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes )
-                                                 .setScroll( cursorTimeout + "m" ).setQuery( qb );;
-
-
-            //we must add a "must" operation to our entity context.
-
-
-            //set our final query
+                                                 .setScroll( cursorTimeout + "m" ).setQuery( qb );
 
 
             if ( logger.isDebugEnabled() ) {
-                        logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
-                                this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
-                        } );
-                    }
-
+                logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
+                        this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
+                } );
+            }
 
 
             final FilterBuilder fb = query.createFilterBuilder();
@@ -287,6 +280,7 @@ public class EsEntityIndexImpl implements EntityIndex {
                 final FieldSortBuilder stringSort =
                         SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( stringSort );
+
                 logger.debug( "   Sort: {} order by {}", stringFieldName, order.toString() );
 
                 final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
@@ -340,12 +334,21 @@ public class EsEntityIndexImpl implements EntityIndex {
             failureMonitor.success();
         }
 
-        SearchHits hits = searchResponse.getHits();
-        logger.debug( "   Hit count: {} Total hits: {}", hits.getHits().length, hits.getTotalHits() );
+        return parseResults( searchResponse, query );
+    }
+
+
+    private CandidateResults parseResults( final SearchResponse searchResponse, final Query query ) {
 
-        List<CandidateResult> candidates = new ArrayList<CandidateResult>();
+        final SearchHits searchHits = searchResponse.getHits();
+        final SearchHit[] hits = searchHits.getHits();
+        final int length = hits.length;
 
-        for ( SearchHit hit : hits.getHits() ) {
+        logger.debug( "   Hit count: {} Total hits: {}", length, searchHits.getTotalHits() );
+
+        List<CandidateResult> candidates = new ArrayList<>( length );
+
+        for ( SearchHit hit : hits ) {
 
             String[] idparts = hit.getId().split( SPLITTER );
             String id = idparts[0];
@@ -396,10 +399,34 @@ public class EsEntityIndexImpl implements EntityIndex {
 
     @Override
     public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
-        Query query = new Query();
-        query.addEqualityFilter( ENTITYID_ID_FIELDNAME, IndexingUtils.idString( id ).toLowerCase() );
-        CandidateResults results = search( scope, SearchTypes.fromTypes( id.getType() ), query );
-        return results;
+
+        //since we don't have paging inputs, there's no point in executing a query for paging.
+
+        final String context = IndexingUtils.createContextName( scope );
+        final SearchTypes searchTypes = SearchTypes.fromTypes( id.getType() );
+
+        final QueryBuilder queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
+
+
+        final SearchRequestBuilder srb =
+                esProvider.getClient().prepareSearch( indexName ).setTypes( searchTypes.getTypeNames() )
+                          .setScroll( cursorTimeout + "m" ).setQuery( queryBuilder );
+
+
+        final SearchResponse searchResponse;
+        try {
+            searchResponse = srb.execute().actionGet();
+        }
+        catch ( Throwable t ) {
+            logger.error( "Unable to communicate with elasticsearch" );
+            failureMonitor.fail( "Unable to execute batch", t );
+            throw t;
+        }
+
+
+        failureMonitor.success();
+
+        return parseResults( searchResponse, new Query(  ) );
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1debf19b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index a1240f1..72058ff 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -72,15 +72,6 @@ public class EntityIndexTest extends BaseIT {
 
     private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
 
-    @ClassRule
-    public static CassandraRule cass = new CassandraRule();
-
-    @Rule
-    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
-
-    @Inject
-    @Rule
-    public MigrationManagerRule migrationManagerRule;
 
     @Inject
     public EntityIndexFactory eif;


[20/27] incubator-usergrid git commit: Fixed AbstractRestIT so that it would no longer use aquireToken calls. Fixed before in TestContextSetup so that we create unique test accounts each time Reformatted and fixed every single test in OrganizationsIT Ad

Posted by to...@apache.org.
Fixed AbstractRestIT so that it would no longer use aquireToken calls.
Fixed before in TestContextSetup  so that we create unique test accounts each time
Reformatted and fixed every single test in OrganizationsIT
Added a uuid when TestUser gets created.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/0fc8818b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/0fc8818b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/0fc8818b

Branch: refs/heads/two-dot-o
Commit: 0fc8818b8135630c7ec6b2725a77a96923c6c97b
Parents: 500a327
Author: grey <gr...@apigee.com>
Authored: Tue Nov 11 07:37:43 2014 -0800
Committer: grey <gr...@apigee.com>
Committed: Tue Nov 11 07:37:43 2014 -0800

----------------------------------------------------------------------
 .../apache/usergrid/rest/AbstractRestIT.java    |  22 +-
 .../apache/usergrid/rest/TestContextSetup.java  |  10 +-
 .../rest/management/OrganizationsIT.java        | 328 +++++++++----------
 .../usergrid/rest/test/security/TestUser.java   |   3 +
 4 files changed, 185 insertions(+), 178 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0fc8818b/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
index e835c33..60944af 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/AbstractRestIT.java
@@ -104,17 +104,17 @@ public abstract class AbstractRestIT extends JerseyTest {
     @Before
     public void acquireToken() throws Exception {
 
-        setupUsers();
-
-        refreshIndex("test-organization", "test-app");
-
-        LOG.info( "acquiring token" );
-        access_token = userToken( "ed@anuff.com", "sesame" );
-        LOG.info( "with token: {}", access_token );
-
-        loginClient();
-
-        refreshIndex("test-organization", "test-app");
+//        setupUsers();
+//
+//        refreshIndex("test-organization", "test-app");
+//
+//        LOG.info( "acquiring token" );
+//        access_token = userToken( "ed@anuff.com", "sesame" );
+//        LOG.info( "with token: {}", access_token );
+//
+//        loginClient();
+//
+//        refreshIndex("test-organization", "test-app");
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0fc8818b/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
index 5580a15..2da1033 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/TestContextSetup.java
@@ -20,6 +20,9 @@ package org.apache.usergrid.rest;
 import org.junit.rules.TestRule;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
+
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
+import org.apache.usergrid.rest.management.OrganizationsIT;
 import org.apache.usergrid.rest.test.resource.TestContext;
 import org.apache.usergrid.rest.test.security.TestAdminUser;
 
@@ -69,7 +72,10 @@ public class TestContextSetup extends TestContext implements TestRule {
         String methodName = description.getMethodName();
         String name = testClass + "." + methodName;
 
-        TestAdminUser testAdmin = new TestAdminUser( name, name + "@usergrid.com", name + "@usergrid.com" );
-        withOrg( name ).withApp( methodName ).withUser( testAdmin ).initAll();
+        TestAdminUser testAdmin = new TestAdminUser( name+UUIDUtils.newTimeUUID(),
+                name + "@usergrid.com"+UUIDUtils.newTimeUUID(),
+                name + "@usergrid.com"+UUIDUtils.newTimeUUID() );
+        withOrg( name+ UUIDUtils.newTimeUUID() ).withApp( methodName + UUIDUtils.newTimeUUID() ).withUser(
+                testAdmin ).initAll();
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0fc8818b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
index 621010c..466610e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/OrganizationsIT.java
@@ -14,6 +14,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+//jnote to self, remaining test failures are due to duplicate org names in the cassandra external. Easily fixed.
 package org.apache.usergrid.rest.management;
 
 
@@ -21,19 +22,19 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
+import java.util.UUID;
 
 import javax.ws.rs.core.MediaType;
 
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.usergrid.management.ApplicationInfo;
-import org.apache.usergrid.management.OrganizationInfo;
-import org.apache.usergrid.management.UserInfo;
 import org.apache.usergrid.persistence.EntityManager;
 import org.apache.usergrid.persistence.entities.User;
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.rest.AbstractRestIT;
 import org.apache.usergrid.rest.TestContextSetup;
 import org.apache.usergrid.rest.management.organizations.OrganizationsResource;
@@ -46,10 +47,6 @@ import com.sun.jersey.api.representation.Form;
 import junit.framework.Assert;
 
 import static junit.framework.Assert.fail;
-import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_ADMIN_USERS_REQUIRE_CONFIRMATION;
-import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_SYSADMIN_APPROVES_ADMIN_USERS;
-import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_SYSADMIN_APPROVES_ORGANIZATIONS;
-import static org.apache.usergrid.management.AccountCreationProps.PROPERTIES_SYSADMIN_EMAIL;
 import static org.apache.usergrid.utils.MapUtils.hashMap;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
@@ -62,65 +59,60 @@ import static org.junit.Assert.assertTrue;
 public class OrganizationsIT extends AbstractRestIT {
     private static final Logger LOG = LoggerFactory.getLogger( OrganizationsIT.class );
 
+    //For Testing OrganizationUpdate
+    @Rule
+    public TestContextSetup context = new TestContextSetup( this );
+
+
     @Test
     public void createOrgAndOwner() throws Exception {
 
-        Map<String, String> originalProperties = getRemoteTestProperties();
+        context.management().tokenGet( context.getActiveUser().getUser(), context.getActiveUser().getPassword() );
+        String username = "createOrgAndOwner" + UUIDUtils.newTimeUUID();
+        String name = username;
+        String password = "password";
+        String orgName = username;
+        String email = username + "@usergrid.com";
 
-        try {
-            setTestProperty( PROPERTIES_SYSADMIN_APPROVES_ADMIN_USERS, "false" );
-            setTestProperty( PROPERTIES_SYSADMIN_APPROVES_ORGANIZATIONS, "false" );
-            setTestProperty( PROPERTIES_ADMIN_USERS_REQUIRE_CONFIRMATION, "false" );
-            setTestProperty( PROPERTIES_SYSADMIN_EMAIL, "sysadmin-1@mockserver.com" );
-
-            Map<String, Object> organizationProperties = new HashMap<String, Object>();
-            organizationProperties.put( "securityLevel", 5 );
-
-            Map payload = hashMap( "email", "test-user-1@mockserver.com" ).map( "username", "test-user-1" )
-                                                                          .map( "name", "Test User" ).map( "password", "password" ).map( "organization", "test-org-1" )
-                                                                          .map( "company", "Apigee" );
-            payload.put( OrganizationsResource.ORGANIZATION_PROPERTIES, organizationProperties );
-
-            JsonNode node = mapper.readTree( resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
-                                                       .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
-
-            assertNotNull( node );
-
-            ApplicationInfo applicationInfo = setup.getMgmtSvc().getApplicationInfo( "test-org-1/sandbox" );
-
-            assertNotNull( applicationInfo );
-
-            Set<String> rolePerms =
-                    setup.getEmf().getEntityManager( applicationInfo.getId() ).getRolePermissions( "guest" );
-            assertNotNull( rolePerms );
-            assertTrue( rolePerms.contains( "get,post,put,delete:/**" ) );
-            logNode( node );
-
-            UserInfo ui = setup.getMgmtSvc().getAdminUserByEmail( "test-user-1@mockserver.com" );
-            EntityManager em = setup.getEmf().getEntityManager( setup.getEmf().getManagementAppId() );
-            User user = em.get( ui.getUuid(), User.class );
-            assertEquals( "Test User", user.getName() );
-            assertEquals( "Apigee", user.getProperty( "company" ));
-
-            OrganizationInfo orgInfo = setup.getMgmtSvc().getOrganizationByName( "test-org-1" );
-            assertEquals( 5L, orgInfo.getProperties().get( "securityLevel" ) );
-
-            node = mapper.readTree( resource().path( "/management/organizations/test-org-1" )
-                                              .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
-                                              .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ));
-            logNode( node );
-            Assert.assertEquals( 5, node.get( "organization" ).get( OrganizationsResource.ORGANIZATION_PROPERTIES )
-                                        .get( "securityLevel" ).asInt() );
-
-            node = mapper.readTree( resource().path( "/management/organizations/test-org-1" )
-                                              .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
-                                              .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ));
-            Assert.assertEquals( 5, node.get( "organization" ).get( OrganizationsResource.ORGANIZATION_PROPERTIES )
-                                        .get( "securityLevel" ).asInt() );
-        }
-        finally {
-            setTestProperties( originalProperties );
-        }
+        Map<String, Object> organizationProperties = new HashMap<String, Object>();
+        organizationProperties.put( "securityLevel", 5 );
+
+        Map payload =
+                hashMap( "email", email ).map( "username", username ).map( "name", name ).map( "password", password )
+                                         .map( "organization", orgName ).map( "company", "Apigee" );
+
+        Map payload2 = hashMap( "grant_type", "password" ).map( "username", username ).map( "password", password );
+        payload.put( OrganizationsResource.ORGANIZATION_PROPERTIES, organizationProperties );
+
+        JsonNode node = mapper.readTree(
+                resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
+                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
+
+        UUID userUuid = UUID.fromString( node.get( "data" ).get( "owner" ).get( "uuid" ).asText() );
+
+        node = mapper.readTree( resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON )
+                                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload2 ) );
+
+        //assertNotNull( node );
+
+        node = mapper.readTree( resource().path( "/management/organizations/" + orgName + "/apps/sandbox" )
+                                          .queryParam( "access_token", node.get( "access_token" ).textValue() )
+                                          .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+                                          .get( String.class ) );
+
+        assertNotNull( node );
+
+        Set<String> rolePerms = setup.getEmf().getEntityManager(
+                UUID.fromString( node.get( "entities" ).get( 0 ).get( "uuid" ).asText() ) )
+                                     .getRolePermissions( "guest" );
+        assertNotNull( rolePerms );
+        assertTrue( rolePerms.contains( "get,post,put,delete:/**" ) );
+        logNode( node );
+
+        EntityManager em = setup.getEmf().getEntityManager( setup.getEmf().getManagementAppId() );
+        User user = em.get( userUuid, User.class );
+        assertEquals( name, user.getName() );
+        assertEquals( "Apigee", user.getProperty( "company" ) );
     }
 
 
@@ -128,101 +120,101 @@ public class OrganizationsIT extends AbstractRestIT {
     public void testCreateDuplicateOrgName() throws Exception {
 
         // create organization with name
+        String timeuuid = UUIDUtils.newTimeUUID().toString();
         Map<String, String> payload =
-                hashMap( "email", "create-duplicate-org@mockserver.com" )
-                        .map( "password", "password" )
-                        .map( "organization", "create-duplicate-orgname-org" );
-        JsonNode node = mapper.readTree( resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
-                                                   .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
+                hashMap( "email", "create-duplicate-org" + timeuuid + "@mockserver.com" ).map( "password", "password" )
+                                                                                         .map( "organization",
+                                                                                                 "create-duplicate-orgname-org"
+                                                                                                         + timeuuid );
+        JsonNode node = mapper.readTree(
+                resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
+                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
 
         logNode( node );
         assertNotNull( node );
 
-        refreshIndex("create-duplicate-orgname-org", "dummy");
+        refreshIndex( "create-duplicate-orgname-org" + timeuuid, "dummy" );
 
         // create another org with that same name, but a different user
-        payload = hashMap( "email", "create-duplicate-org2@mockserver.com" )
-                .map( "username", "create-dupe-orgname2" )
-                .map( "password", "password" )
-                .map( "organization", "create-duplicate-orgname-org" );
+        payload = hashMap( "email", "create-duplicate-org2@mockserver.com" ).map( "username", "create-dupe-orgname2" )
+                                                                            .map( "password", "password" )
+                                                                            .map( "organization",
+                                                                                    "create-duplicate-orgname-org"
+                                                                                            + timeuuid );
         try {
             node = mapper.readTree( resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
-                                              .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
+                                              .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
         }
         catch ( Exception ex ) {
         }
 
-        refreshIndex("create-duplicate-orgname-org", "dummy");
+        refreshIndex( "create-duplicate-orgname-org" + timeuuid, "dummy" );
 
         // now attempt to login as the user for the second organization
-        payload = hashMap( "grant_type", "password" )
-                .map( "username", "create-dupe-orgname2" )
-                .map( "password", "password" );
+        payload = hashMap( "grant_type", "password" ).map( "username", "create-dupe-orgname2" )
+                                                     .map( "password", "password" );
         try {
             node = mapper.readTree( resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON )
-                                              .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
+                                              .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
             fail( "Should not have created user" );
         }
         catch ( Exception ex ) {
         }
         logNode( node );
 
-        refreshIndex("create-duplicate-orgname-org", "dummy");
+        refreshIndex( "create-duplicate-orgname-org" + timeuuid, "dummy" );
 
-        payload = hashMap( "username", "create-duplicate-org@mockserver.com" )
-                .map( "grant_type", "password" )
-                .map( "password", "password" );
+        payload = hashMap( "username", "create-duplicate-org@mockserver.com" ).map( "grant_type", "password" )
+                                                                              .map( "password", "password" );
         node = mapper.readTree( resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON )
-                                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
+                                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
         logNode( node );
     }
 
+
     @Test
     public void testCreateDuplicateOrgEmail() throws Exception {
 
+        String timeUuid = UUIDUtils.newTimeUUID().toString();
         Map<String, String> payload =
-                hashMap( "email", "duplicate-email@mockserver.com" )
-                        .map( "password", "password" )
-                        .map( "organization", "very-nice-org" );
+                hashMap( "email", "duplicate-email" + timeUuid + "@mockserver.com" ).map( "password", "password" )
+                                                                                    .map( "organization",
+                                                                                            "very-nice-org"
+                                                                                                    + timeUuid );
 
-        JsonNode node = mapper.readTree( resource().path( "/management/organizations" )
-                                                   .accept( MediaType.APPLICATION_JSON )
-                                                   .type( MediaType.APPLICATION_JSON_TYPE )
-                                                   .post( String.class, payload ));
+        JsonNode node = mapper.readTree(
+                resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
+                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
 
         logNode( node );
         assertNotNull( node );
 
-        payload = hashMap( "email", "duplicate-email@mockserver.com" )
-                .map( "username", "anotheruser" )
-                .map( "password", "password" )
-                .map( "organization", "not-so-nice-org" );
+        payload = hashMap( "email", "duplicate-email" + timeUuid + "@mockserver.com" ).map( "username", "anotheruser" )
+                                                                                      .map( "password", "password" )
+                                                                                      .map( "organization",
+                                                                                              "not-so-nice-org"
+                                                                                                      + timeUuid );
 
         boolean failed = false;
         try {
-            node = mapper.readTree( resource().path( "/management/organizations" )
-                                              .accept( MediaType.APPLICATION_JSON )
-                                              .type( MediaType.APPLICATION_JSON_TYPE )
-                                              .post( String.class, payload ));
+            node = mapper.readTree( resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
+                                              .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
         }
         catch ( UniformInterfaceException ex ) {
             Assert.assertEquals( 400, ex.getResponse().getStatus() );
             JsonNode errorJson = ex.getResponse().getEntity( JsonNode.class );
-            Assert.assertEquals( "duplicate_unique_property_exists", errorJson.get("error").asText());
+            Assert.assertEquals( "duplicate_unique_property_exists", errorJson.get( "error" ).asText() );
             failed = true;
         }
-        Assert.assertTrue(failed);
+        Assert.assertTrue( failed );
 
-        refreshIndex("test-organization", "test-app");
+        refreshIndex( "test-organization", "test-app" );
 
-        payload = hashMap( "grant_type", "password" )
-                .map( "username", "create-dupe-orgname2" )
-                .map( "password", "password" );
+        payload = hashMap( "grant_type", "password" ).map( "username", "create-dupe-orgname2" )
+                                                     .map( "password", "password" );
         try {
-            node = mapper.readTree( resource().path( "/management/token" )
-                                              .accept( MediaType.APPLICATION_JSON )
-                                              .type( MediaType.APPLICATION_JSON_TYPE )
-                                              .post( String.class, payload ));
+            node = mapper.readTree( resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON )
+                                              .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
             fail( "Should not have created user" );
         }
         catch ( Exception ex ) {
@@ -230,27 +222,31 @@ public class OrganizationsIT extends AbstractRestIT {
 
         logNode( node );
 
-        refreshIndex("test-organization", "test-app");
+        refreshIndex( "test-organization", "test-app" );
 
-        payload = hashMap( "username", "duplicate-email@mockserver.com" )
-                .map( "grant_type", "password" )
-                .map( "password", "password" );
-        node = mapper.readTree( resource().path( "/management/token" )
-                                          .accept( MediaType.APPLICATION_JSON )
-                                          .type( MediaType.APPLICATION_JSON_TYPE )
-                                          .post( String.class, payload ));
+        payload =
+                hashMap( "username", "duplicate-email" + timeUuid + "@mockserver.com" ).map( "grant_type", "password" )
+                                                                                       .map( "password", "password" );
+        node = mapper.readTree( resource().path( "/management/token" ).accept( MediaType.APPLICATION_JSON )
+                                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
         logNode( node );
     }
 
+
     @Test
     public void testOrgPOSTParams() throws IOException {
-        JsonNode node = mapper.readTree( resource().path( "/management/organizations" ).queryParam( "organization", "testOrgPOSTParams" )
-                                                   .queryParam( "username", "testOrgPOSTParams" ).queryParam( "grant_type", "password" )
-                                                   .queryParam( "email", "testOrgPOSTParams@apigee.com" ).queryParam( "name", "testOrgPOSTParams" )
+        UUID timeUuid = UUIDUtils.newTimeUUID();
+        JsonNode node = mapper.readTree( resource().path( "/management/organizations" )
+                                                   .queryParam( "organization", "testOrgPOSTParams" + timeUuid )
+                                                   .queryParam( "username", "testOrgPOSTParams" + timeUuid )
+                                                   .queryParam( "grant_type", "password" ).queryParam( "email",
+                        "testOrgPOSTParams" + timeUuid + "@apigee.com" + timeUuid )
+                                                   .queryParam( "name", "testOrgPOSTParams" )
                                                    .queryParam( "password", "password" )
 
-                                                   .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_FORM_URLENCODED )
-                                                   .post( String.class ));
+                                                   .accept( MediaType.APPLICATION_JSON )
+                                                   .type( MediaType.APPLICATION_FORM_URLENCODED )
+                                                   .post( String.class ) );
 
         assertEquals( "ok", node.get( "status" ).asText() );
     }
@@ -259,33 +255,36 @@ public class OrganizationsIT extends AbstractRestIT {
     @Test
     public void testOrgPOSTForm() throws IOException {
 
+        UUID timeUuid = UUIDUtils.newTimeUUID();
         Form form = new Form();
-        form.add( "organization", "testOrgPOSTForm" );
-        form.add( "username", "testOrgPOSTForm" );
+        form.add( "organization", "testOrgPOSTForm" + timeUuid );
+        form.add( "username", "testOrgPOSTForm" + timeUuid );
         form.add( "grant_type", "password" );
-        form.add( "email", "testOrgPOSTForm@apigee.com" );
+        form.add( "email", "testOrgPOSTForm" + timeUuid + "@apigee.com" );
         form.add( "name", "testOrgPOSTForm" );
         form.add( "password", "password" );
 
-        JsonNode node = mapper.readTree( resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
-                                                   .type( MediaType.APPLICATION_FORM_URLENCODED ).post( String.class, form ));
+        JsonNode node = mapper.readTree(
+                resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
+                          .type( MediaType.APPLICATION_FORM_URLENCODED ).post( String.class, form ) );
 
         assertEquals( "ok", node.get( "status" ).asText() );
     }
 
+
     @Test
     public void noOrgDelete() throws IOException {
 
 
-        String mgmtToken = adminToken();
+        String mgmtToken = context.getActiveUser().getToken();
 
         ClientResponse.Status status = null;
         JsonNode node = null;
 
         try {
-            node = mapper.readTree( resource().path( "/test-organization" ).queryParam( "access_token", mgmtToken )
-                                              .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
-                                              .delete( String.class ));
+            node = mapper.readTree( resource().path( context.getOrgName() ).queryParam( "access_token", mgmtToken )
+                                              .accept( MediaType.APPLICATION_JSON )
+                                              .type( MediaType.APPLICATION_JSON_TYPE ).delete( String.class ) );
         }
         catch ( UniformInterfaceException uie ) {
             status = uie.getResponse().getClientResponseStatus();
@@ -294,21 +293,21 @@ public class OrganizationsIT extends AbstractRestIT {
         assertEquals( ClientResponse.Status.NOT_IMPLEMENTED, status );
     }
 
+
     @Test
     public void testCreateOrgUserAndReturnCorrectUsername() throws Exception {
 
-        String mgmtToken = superAdminToken();
 
-        Map<String, String> payload = hashMap( "username", "test-user-2" )
-                .map("name", "Test User 2")
-                .map("email", "test-user-2@mockserver.com")
-                .map( "password", "password" );
+        String mgmtToken = context.getActiveUser().getToken();
 
-        JsonNode node = mapper.readTree( resource().path( "/management/organizations/test-organization/users" )
-                                                   .queryParam( "access_token", mgmtToken )
-                                                   .accept( MediaType.APPLICATION_JSON )
-                                                   .type( MediaType.APPLICATION_JSON_TYPE )
-                                                   .post( String.class, payload ));
+        Map<String, String> payload = hashMap( "username", "test-user-2" ).map( "name", "Test User 2" )
+                                                                          .map( "email", "test-user-2@mockserver.com" )
+                                                                          .map( "password", "password" );
+
+        JsonNode node = mapper.readTree(
+                resource().path( "/management/organizations/" + context.getOrgName() + "/users" )
+                          .queryParam( "access_token", mgmtToken ).accept( MediaType.APPLICATION_JSON )
+                          .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ) );
 
         logNode( node );
         assertNotNull( node );
@@ -326,53 +325,52 @@ public class OrganizationsIT extends AbstractRestIT {
         assertEquals( "test-user-2@mockserver.com", email );
     }
 
-    //For Testing OrganizationUpdate
-    @Rule
-    public TestContextSetup context = new TestContextSetup( this );
-
 
     @Test
     public void testOrganizationUpdate() throws Exception {
+        String accessToken = context.getActiveUser().getToken();
         Map<String, Object> properties = new HashMap<String, Object>();
         properties.put( "securityLevel", 5 );
+        Map payload = new HashMap();
+        payload.put( OrganizationsResource.ORGANIZATION_PROPERTIES, properties );
 
-        Map payload = hashMap( "email", "test-user-1@organizationresourceit.testorganizationupdate.com" )
-                .map( "username", "organizationresourceit.testorganizationupdate.test-user-1" )
-                .map( "name", "organizationresourceit.testorganizationupdate" ).map( "password", "password" )
-                .map( "organization", "organizationresourceit.testorganizationupdate.test-org-1" )
-                .map( "company", "Apigee" );
 
+        //update the organizations
+        mapper.readTree( resource().path( "/management/organizations/" + context.getOrgName() )
+                                   .queryParam( "access_token", accessToken ).accept( MediaType.APPLICATION_JSON )
+                                   .type( MediaType.APPLICATION_JSON_TYPE ).put( String.class, payload ) );
 
-        payload.put( OrganizationsResource.ORGANIZATION_PROPERTIES, properties );
 
-        JsonNode node = mapper.readTree( resource().path( "/management/organizations" ).accept( MediaType.APPLICATION_JSON )
-                                                   .type( MediaType.APPLICATION_JSON_TYPE ).post( String.class, payload ));
-        assertNotNull( node );
+        refreshIndex( context.getOrgName(), context.getAppName() );
 
-        refreshIndex(context.getOrgName(), context.getAppName());
+        //get the organization
+        JsonNode node = mapper.readTree( resource().path( "/management/organizations/" + context.getOrgName() )
+                                                   .queryParam( "access_token", accessToken )
+                                                   .accept( MediaType.APPLICATION_JSON )
+                                                   .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ) );
 
-        OrganizationInfo orgInfo =
-                setup.getMgmtSvc().getOrganizationByName( "organizationresourceit.testorganizationupdate.test-org-1" );
-        assertEquals( 5L, orgInfo.getProperties().get( "securityLevel" ) );
+        assertEquals( 5L, node.get( "organization" ).get( "properties" ).get( "securityLevel" )
+                              .asLong() );//orgInfo.getProperties().get( "securityLevel" ) );
 
         payload = new HashMap();
         properties.put( "securityLevel", 6 );
         payload.put( OrganizationsResource.ORGANIZATION_PROPERTIES, properties );
 
-        node = mapper.readTree( resource().path( "/management/organizations/organizationresourceit.testorganizationupdate.test-org-1" )
-                                          .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
-                                          .type( MediaType.APPLICATION_JSON_TYPE ).put( String.class, payload ));
+        node = mapper.readTree( resource().path( "/management/organizations/" + context.getOrgName() )
+                                          .queryParam( "access_token", accessToken )
+                                          .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+                                          .put( String.class, payload ) );
         logNode( node );
 
-        refreshIndex(context.getOrgName(), context.getAppName());
+        refreshIndex( context.getOrgName(), context.getAppName() );
 
-        node = mapper.readTree( resource().path( "/management/organizations/organizationresourceit.testorganizationupdate.test-org-1" )
-                                          .queryParam( "access_token", superAdminToken() ).accept( MediaType.APPLICATION_JSON )
-                                          .type( MediaType.APPLICATION_JSON_TYPE ).get( String.class ));
+        node = mapper.readTree( resource().path( "/management/organizations/" + context.getOrgName() )
+                                          .queryParam( "access_token", accessToken )
+                                          .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
+                                          .get( String.class ) );
         logNode( node );
         Assert.assertEquals( 6,
                 node.get( "organization" ).get( OrganizationsResource.ORGANIZATION_PROPERTIES ).get( "securityLevel" )
                     .asInt() );
     }
-
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/0fc8818b/stack/rest/src/test/java/org/apache/usergrid/rest/test/security/TestUser.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/test/security/TestUser.java b/stack/rest/src/test/java/org/apache/usergrid/rest/test/security/TestUser.java
index 871d46e..b322d90 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/test/security/TestUser.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/test/security/TestUser.java
@@ -20,6 +20,8 @@ package org.apache.usergrid.rest.test.security;
 import java.util.UUID;
 
 import com.fasterxml.jackson.databind.JsonNode;
+
+import org.apache.usergrid.persistence.index.utils.UUIDUtils;
 import org.apache.usergrid.rest.test.resource.TestContext;
 
 
@@ -43,6 +45,7 @@ public abstract class TestUser {
         this.user = user;
         this.password = password;
         this.email = email;
+        this.uuid = UUIDUtils.newTimeUUID();
     }
 
 


[26/27] incubator-usergrid git commit: Merge remote-tracking branch 'origin/two-dot-o' into es-type-updates-ryan-external

Posted by to...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into es-type-updates-ryan-external


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/28eb2cd9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/28eb2cd9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/28eb2cd9

Branch: refs/heads/two-dot-o
Commit: 28eb2cd9fe6b1fbccc0ddd1fd9693bddd42edd5c
Parents: e7b7758 84b8365
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 12 12:11:07 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 12 12:11:07 2014 -0700

----------------------------------------------------------------------
 .../datagenerators/FeederGenerator.scala        | 17 +++------
 .../usergrid/scenarios/DeviceScenarios.scala    | 34 +++++++++--------
 .../usergrid/scenarios/UserScenarios.scala      | 40 +++++++++++---------
 .../org/apache/usergrid/settings/Settings.scala |  5 ++-
 .../simulations/ConnectionsSimulation.scala     |  2 +-
 .../simulations/GetEntitySimulation.scala       |  2 +-
 .../PushNotificationTargetUserSimulation.scala  |  2 +-
 7 files changed, 51 insertions(+), 51 deletions(-)
----------------------------------------------------------------------



[09/27] incubator-usergrid git commit: Fixed read to include compile phase, which is necessary b/c gatling does not execute this phase as a dependency

Posted by to...@apache.org.
Fixed read to include compile phase, which is necessary b/c gatling does not execute this phase as a dependency


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/be6c3c2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/be6c3c2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/be6c3c2d

Branch: refs/heads/two-dot-o
Commit: be6c3c2d1ba3e7f5d5708ab09a556e11f2da7184
Parents: 8bab4e7
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 7 15:34:13 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 7 15:34:13 2014 -0700

----------------------------------------------------------------------
 stack/loadtests/README.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/be6c3c2d/stack/loadtests/README.md
----------------------------------------------------------------------
diff --git a/stack/loadtests/README.md b/stack/loadtests/README.md
index b246d04..7f63b18 100644
--- a/stack/loadtests/README.md
+++ b/stack/loadtests/README.md
@@ -3,7 +3,7 @@ Gatling will run through mvn
 
 1. navigate to loadtests under stack
 
-1. execute mvn gatling:execute with these options
+1. execute mvn compile gatling:execute with these options
 
 	*Required
 	> -Dthrottle={maxReqsSec} -Dduration={durationInSecs} -Dorg={org}  -Dapp={appName} -Dbaseurl={uriNoProceedingSlash} -DnumEntities={numberOfEntitiesYouWantToCreateInteger} -DnumUsers={totalUsersInteger} -DrampTime={rampTimeIntegerSeconds} -DadminUser={username} -DadminPassword={pwd} -Dgatling.simulationClass={simulationFQDN}
@@ -13,7 +13,7 @@ Gatling will run through mvn
 	>-DpushNotifier={notifierName} -DpushProvider={noop_apns_or_gcm} 
 
 	So running will look something like this
-	>mvn gatling:execute -Dthrottle=100 -Dduration=300 -Dorg=usergrid  -Dapp=load -Dbaseurl=http://load.usergrid.com -DnumEntities=300 -DnumUsers=600 -DrampTime=30 -DadminUser=usergrid -DadminPassword=test -Dgatling.simulationClass=org.apache.usergrid.simulations.AppSimulation
+	>mvn compile gatling:execute -Dthrottle=100 -Dduration=300 -Dorg=usergrid  -Dapp=load -Dbaseurl=http://load.usergrid.com -DnumEntities=300 -DnumUsers=600 -DrampTime=30 -DadminUser=usergrid -DadminPassword=test -Dgatling.simulationClass=org.apache.usergrid.simulations.AppSimulation
 	
 	Setting the rampTime => Injects a given number of users with a linear ramp over a given duration.
 
@@ -21,4 +21,4 @@ Gatling will run through mvn
 	
 	Also see http://gatling.io/docs/2.0.2/general/simulation_setup.html
 	
-	Additional docs can be found here http://gatling.io/docs/2.0.2/
\ No newline at end of file
+	Additional docs can be found here http://gatling.io/docs/2.0.2/


[06/27] incubator-usergrid git commit: External tests working except for old static org/app tests

Posted by to...@apache.org.
External tests working except for old static org/app tests


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d4ecbe17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d4ecbe17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d4ecbe17

Branch: refs/heads/two-dot-o
Commit: d4ecbe17a3063f902d211c8a02e9ba3c1e8de284
Parents: ef9e94e
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 7 09:20:42 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 7 09:20:42 2014 -0700

----------------------------------------------------------------------
 .../config/src/test/resources/usergrid-test.properties |  2 +-
 .../src/test/resources/usergrid-custom-test.properties |  4 ++--
 .../migration/data/MigrationInfoSerializationImpl.java |  2 +-
 .../migration/data/MigrationInfoSerializationTest.java | 13 ++++++++++++-
 .../src/test/resources/dynamic-test.properties         |  2 +-
 .../src/test/resources/usergrid-UNIT.properties        |  2 +-
 .../src/test/resources/usergrid-custom-test.properties |  4 ++--
 .../src/test/resources/usergrid-custom-test.properties |  4 ++--
 .../src/test/resources/usergrid-custom-test.properties |  2 +-
 9 files changed, 23 insertions(+), 12 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index fc654f9..7d1da1e 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -58,7 +58,7 @@ cassandra.writecl=QUORUM
 #Keyspace to use for locking
 cassandra.lock.keyspace=Locks
 
-elasticsearch.startup=forked
+elasticsearch.startup=external
 elasticsearch.cluster_name=test_cluster
 
 #Properties to control the number of buckets in the index.

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/core/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/core/src/test/resources/usergrid-custom-test.properties b/stack/core/src/test/resources/usergrid-custom-test.properties
index 6917df0..a1ff5b2 100644
--- a/stack/core/src/test/resources/usergrid-custom-test.properties
+++ b/stack/core/src/test/resources/usergrid-custom-test.properties
@@ -14,8 +14,8 @@
 
 # these settings allow tests to run and consistently pass on 16GB MacBook Pro 
 # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml) 
-cassandra.startup=embedded
-elasticsearch.startup=embedded
+cassandra.startup=external
+elasticsearch.startup=external
 cassandra.timeout=2000
 cassandra.connections=1000
 hystrix.threadpool.graph_user.coreSize=50

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
index 0460e33..5dbadee 100644
--- a/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
+++ b/stack/corepersistence/common/src/main/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationImpl.java
@@ -62,7 +62,7 @@ public class MigrationInfoSerializationImpl implements MigrationInfoSerializatio
     private static final StringSerializer STRING_SERIALIZER = StringSerializer.get();
 
 
-    private static final MultiTennantColumnFamily<ScopedRowKey<String>, String> CF_MIGRATION_INFO =
+    public static final MultiTennantColumnFamily<ScopedRowKey<String>, String> CF_MIGRATION_INFO =
             new MultiTennantColumnFamily<>( "Data_Migration_Info", ROW_KEY_SER, STRING_SERIALIZER );
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
index 4742496..598d107 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/migration/data/MigrationInfoSerializationTest.java
@@ -27,6 +27,8 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
+import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
+import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
 import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.guice.MigrationManagerRule;
 import org.apache.usergrid.persistence.core.guice.TestCommonModule;
@@ -35,6 +37,7 @@ import org.apache.usergrid.persistence.core.test.UseModules;
 
 import com.google.inject.Inject;
 import com.netflix.astyanax.Keyspace;
+import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNull;
@@ -49,6 +52,9 @@ public class MigrationInfoSerializationTest {
     public MigrationManagerRule migrationManagerRule;
 
     @Inject
+    public MigrationManager migrationManager;
+
+    @Inject
     protected Keyspace keyspace;
 
 
@@ -60,7 +66,12 @@ public class MigrationInfoSerializationTest {
 
 
     @Test
-    public void basicTest(){
+    public void basicTest() throws ConnectionException, MigrationException {
+
+        //drop the column family, then run setup
+        keyspace.dropColumnFamily( MigrationInfoSerializationImpl.CF_MIGRATION_INFO.getName() );
+
+         migrationManager.migrate();
 
         //test getting nothing works
         final String emptyStatus = migrationInfoSerialization.getStatusMessage();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
index bf862de..3d4582f 100644
--- a/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/dynamic-test.properties
@@ -10,7 +10,7 @@ cassandra.timeout=5000
 
 index.query.limit.default=10
 elasticsearch.indexname=Usergrid
-elasticsearch.startup=embedded
+elasticsearch.startup=external
 elasticsearch.force-refresh=false
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
index 032d5c5..d7d3909 100644
--- a/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
+++ b/stack/corepersistence/queryindex/src/test/resources/usergrid-UNIT.properties
@@ -13,7 +13,7 @@ collections.keyspace.strategy.options=replication_factor:1
 collections.keyspace.strategy.class=org.apache.cassandra.locator.SimpleStrategy
 collection.stage.transient.timeout=6
 
-elasticsearch.startup=forked
+elasticsearch.startup=external
 elasticsearch.cluster_name=usergrid
 elasticsearch.index_prefix=usergrid
 elasticsearch.hosts=127.0.0.1

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/rest/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/resources/usergrid-custom-test.properties b/stack/rest/src/test/resources/usergrid-custom-test.properties
index 8d39006..ef615a2 100644
--- a/stack/rest/src/test/resources/usergrid-custom-test.properties
+++ b/stack/rest/src/test/resources/usergrid-custom-test.properties
@@ -17,11 +17,11 @@
 tomcat.startup=embedded
 tomcat.threads=200
 
-cassandra.startup=forked
+cassandra.startup=external
 cassandra.timeout=2000
 cassandra.connections=800
 
-elasticsearch.startup=forked
+elasticsearch.startup=external
 
 hystrix.threadpool.graph_user.coreSize=1200
 hystrix.threadpool.graph_async.coreSize=1200

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/services/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/services/src/test/resources/usergrid-custom-test.properties b/stack/services/src/test/resources/usergrid-custom-test.properties
index c39fe3e..5481a43 100644
--- a/stack/services/src/test/resources/usergrid-custom-test.properties
+++ b/stack/services/src/test/resources/usergrid-custom-test.properties
@@ -14,8 +14,8 @@
 
 # these settings allow tests to run and consistently pass on 16GB MacBook Pro 
 # with ug.heapmax=5000m and ug.heapmin=3000m (set in Maven settings.xml) 
-cassandra.startup=embedded
-elasticsearch.startup=embedded
+cassandra.startup=external
+elasticsearch.startup=external
 cassandra.timeout=2000
 cassandra.connections=1000
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/d4ecbe17/stack/test-utils/src/test/resources/usergrid-custom-test.properties
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/test/resources/usergrid-custom-test.properties b/stack/test-utils/src/test/resources/usergrid-custom-test.properties
index 0ff85ff..5a463aa 100644
--- a/stack/test-utils/src/test/resources/usergrid-custom-test.properties
+++ b/stack/test-utils/src/test/resources/usergrid-custom-test.properties
@@ -15,4 +15,4 @@
 # limitations under the License.
 
 
-cassandra.startup=forked
\ No newline at end of file
+cassandra.startup=external


[18/27] incubator-usergrid git commit: Fixed keyspace options to match new setup. Previous did not work in real cluster since optiosn are different for replication strategy.

Posted by to...@apache.org.
Fixed keyspace options to match new setup.  Previous did not work in real cluster since optiosn are different for replication strategy.

Cleaned up usage in CassandraResource to have single code path

Removed after logic.  In parallel execution this removes keyspaces as other tests are running


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/500a3276
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/500a3276
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/500a3276

Branch: refs/heads/two-dot-o
Commit: 500a3276544e780226f26c4b887b49eadc6da7be
Parents: a103e7d
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 17:47:37 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 17:47:37 2014 -0700

----------------------------------------------------------------------
 .../main/resources/usergrid-default.properties  |   2 +-
 .../src/test/resources/usergrid-test.properties |   2 +-
 .../apache/usergrid/ConcurrentCoreITSuite.java  |   2 +-
 .../usergrid/ConcurrentCoreIteratorITSuite.java |   2 +-
 .../org/apache/usergrid/CoreApplication.java    |   3 +-
 .../java/org/apache/usergrid/CoreITSuite.java   |   2 +-
 .../usergrid/cassandra/CassandraResource.java   | 272 ++++---------------
 .../cassandra/CassandraResourceTest.java        |   7 +-
 8 files changed, 66 insertions(+), 226 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/config/src/main/resources/usergrid-default.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/main/resources/usergrid-default.properties b/stack/config/src/main/resources/usergrid-default.properties
index a2d73be..c840d21 100644
--- a/stack/config/src/main/resources/usergrid-default.properties
+++ b/stack/config/src/main/resources/usergrid-default.properties
@@ -84,7 +84,7 @@ cassandra.keyspace.strategy=org.apache.cassandra.locator.SimpleStrategy
 #cassandra.keyspace.strategy.options.replication_factor=1
 #cassandra.keyspace.strategy.options.us-east=1
 
-cassandra.keyspace.replication=1
+cassandra.keyspace.replication=replication_factor:1
 
 cassandra.username=
 cassandra.password=

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index 7d1da1e..cfa1f2f 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -42,7 +42,7 @@ cassandra.keyspace.strategy=org.apache.cassandra.locator.SimpleStrategy
 #cassandra.keyspace.strategy.options.replication_factor=1
 #cassandra.keyspace.strategy.options.us-east=1
 
-cassandra.keyspace.replication=1
+cassandra.keyspace.replication=replication_factor:1
 
 cassandra.username=
 cassandra.password=

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreITSuite.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreITSuite.java b/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreITSuite.java
index e598a71..d981ee9 100644
--- a/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreITSuite.java
+++ b/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreITSuite.java
@@ -47,5 +47,5 @@ import org.apache.usergrid.system.UsergridSystemMonitorIT;
 @Concurrent()
 public class ConcurrentCoreITSuite {
     @ClassRule
-    public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( "coreManager" );
+    public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( );
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java b/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
index fec436e..273c6df 100644
--- a/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
+++ b/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
@@ -76,7 +76,7 @@ import org.apache.usergrid.persistence.query.SingleOrderBySameRangeScanLessThanE
 public class ConcurrentCoreIteratorITSuite {
 
     @ClassRule
-    public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( "coreManager" );
+    public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( );
     
     @ClassRule
     public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
index fd8ca9b..f6d9132 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreApplication.java
@@ -35,6 +35,7 @@ import org.apache.usergrid.persistence.Results;
 import static junit.framework.Assert.assertNotNull;
 import org.apache.usergrid.persistence.EntityRef;
 import org.apache.usergrid.persistence.SimpleEntityRef;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
 
 public class CoreApplication implements Application, TestRule {
@@ -145,7 +146,7 @@ public class CoreApplication implements Application, TestRule {
 
 
     protected void before( Description description ) throws Exception {
-        orgName = description.getClassName();
+        orgName = description.getClassName()+ UUIDGenerator.newTimeUUID();
         appName = description.getMethodName();
         id = setup.createApplication( orgName, appName );
         assertNotNull( id );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java b/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
index 41997ae..ccdcd98 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
@@ -48,7 +48,7 @@ import org.apache.usergrid.system.UsergridSystemMonitorIT;
 public class CoreITSuite {
     
     @ClassRule
-    public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( "coreManager" );
+    public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( );
 
     @ClassRule
     public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
index 3de0144..91acf79 100644
--- a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
@@ -17,7 +17,6 @@
 package org.apache.usergrid.cassandra;
 
 
-import java.io.File;
 import java.io.IOException;
 import java.util.Properties;
 
@@ -27,10 +26,7 @@ import org.slf4j.LoggerFactory;
 import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.support.ClassPathXmlApplicationContext;
 
-import org.apache.commons.io.FileUtils;
 import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.math.RandomUtils;
 
 
 /**
@@ -55,22 +51,14 @@ public class CassandraResource extends ExternalResource {
     public static final String DEFAULT_HOST = "127.0.0.1";
 
     public static final String PROPERTIES_FILE = "project.properties";
-    public static final String TARGET_DIRECTORY_KEY = "target.directory";
-    public static final String DATA_FILE_DIR_KEY = "data_file_directories";
-    public static final String COMMIT_FILE_DIR_KEY = "commitlog_directory";
-    public static final String SAVED_CACHES_DIR_KEY = "saved_caches_directory";
 
     public static final String NATIVE_TRANSPORT_PORT_KEY = "native_transport_port";
     public static final String RPC_PORT_KEY = "rpc_port";
     public static final String STORAGE_PORT_KEY = "storage_port";
     public static final String SSL_STORAGE_PORT_KEY = "ssl_storage_port";
-    public static final String JAMM_PATH = "jamm.path";
 
     private static final Object lock = new Object();
 
-    private final File tempDir;
-    private final String schemaManagerName;
-
     private boolean initialized = false;
     private int rpcPort = DEFAULT_RPC_PORT;
     private int storagePort = DEFAULT_STORAGE_PORT;
@@ -81,43 +69,23 @@ public class CassandraResource extends ExternalResource {
     private SchemaManager schemaManager;
 
     private static CassandraResource instance;
-    private Thread shutdown;
 
     private static Properties properties = null;
 
 
     /**
-     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the default SchemaManager for
-     * Cassandra.
-     */
-    @SuppressWarnings( "UnusedDeclaration" )
-    CassandraResource() throws IOException {
-        this( null, DEFAULT_RPC_PORT, DEFAULT_STORAGE_PORT, DEFAULT_SSL_STORAGE_PORT, DEFAULT_NATIVE_TRANSPORT_PORT );
-    }
-
-
-    /**
      * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the specified SchemaManager for
      * Cassandra.
      */
-    CassandraResource( String schemaManagerName, int rpcPort, int storagePort, int sslStoragePort,
-                       int nativeTransportPort ) {
+    CassandraResource( int rpcPort, int storagePort, int sslStoragePort, int nativeTransportPort ) {
         LOG.info( "Creating CassandraResource using {} for the ClassLoader.",
                 Thread.currentThread().getContextClassLoader() );
 
-        this.schemaManagerName = schemaManagerName;
         this.rpcPort = rpcPort;
         this.storagePort = storagePort;
         this.sslStoragePort = sslStoragePort;
         this.nativeTransportPort = nativeTransportPort;
 
-        try {
-            this.tempDir = getTempDirectory();
-        }
-        catch ( IOException e ) {
-            LOG.error( "Failed to create temporary directory for Cassandra instance.", e );
-            throw new RuntimeException( e );
-        }
 
         try {
             String[] locations = { "usergrid-properties-context.xml" };
@@ -125,7 +93,6 @@ public class CassandraResource extends ExternalResource {
 
             Properties properties = ( Properties ) appContext.getBean( "properties" );
             properties.putAll( ArrayUtils.toMap( this.getProjectProperties().entrySet().toArray( new Object[] { } ) ) );
-            String forkString = properties.getProperty( "cassandra.startup" );
         }
         catch ( Exception ex ) {
             throw new RuntimeException( "Error getting properties", ex );
@@ -134,16 +101,6 @@ public class CassandraResource extends ExternalResource {
 
 
     /**
-     * Creates a Cassandra starting ExternalResource for JUnit test cases which uses the specified SchemaManager for
-     * Cassandra.
-     */
-    public CassandraResource( int rpcPort, int storagePort, int sslStoragePort, int nativeTransportPort )
-            throws IOException {
-        this( null, rpcPort, storagePort, sslStoragePort, nativeTransportPort );
-    }
-
-
-    /**
      * Gets the rpcPort for Cassandra.
      *
      * @return the rpc_port (in yaml file) used by Cassandra
@@ -179,17 +136,6 @@ public class CassandraResource extends ExternalResource {
 
 
     /**
-     * Gets the temporary directory created for this CassandraResource.
-     *
-     * @return the temporary directory
-     */
-    @SuppressWarnings( "UnusedDeclaration" )
-    public File getTemporaryDirectory() {
-        return tempDir;
-    }
-
-
-    /**
      * Gets a bean from the application context.
      *
      * @param requiredType the type of the bean
@@ -227,15 +173,6 @@ public class CassandraResource extends ExternalResource {
     }
 
 
-    @Override
-    public String toString() {
-        return "\n" + "cassandra.yaml = " + new File( tempDir, "cassandra.yaml" ) + "\n" + RPC_PORT_KEY + " = "
-                + rpcPort + "\n" + STORAGE_PORT_KEY + " = " + storagePort + "\n" + SSL_STORAGE_PORT_KEY + " = "
-                + sslStoragePort + "\n" + NATIVE_TRANSPORT_PORT_KEY + " = " + nativeTransportPort + "\n"
-                + DATA_FILE_DIR_KEY + " = " + new File( tempDir, "data" ).toString() + "\n" + COMMIT_FILE_DIR_KEY
-                + " = " + new File( tempDir, "commitlog" ).toString() + "\n" + SAVED_CACHES_DIR_KEY + " = " + new File(
-                tempDir, "saved_caches" ).toString() + "\n";
-    }
 
 
     /**
@@ -276,29 +213,10 @@ public class CassandraResource extends ExternalResource {
             }
 
             startCassandraExternal();
-
-            //            if ( forkCassandra ) {
-            //                startCassandraForked();
-            //            } else if (externalCassandra) {
-            //
-            //            }else {
-            //
-            //                startCassandraEmbedded();
-            //            }
         }
     }
 
 
-    private void addShutdownHook() {
-        Runtime.getRuntime().addShutdownHook( new Thread() {
-            @Override
-            public void run() {
-                after();
-            }
-        } );
-    }
-
-
     private void startCassandraExternal() throws Throwable {
         LOG.info( "-------------------------------------------------------------------" );
         LOG.info( "Initializing External Cassandra" );
@@ -310,10 +228,8 @@ public class CassandraResource extends ExternalResource {
         String[] locations = { "usergrid-test-context.xml" };
         applicationContext = new ClassPathXmlApplicationContext( locations );
         applicationContext.refresh();
-        loadSchemaManager( schemaManagerName );
+        loadSchemaManager();
         initialized = true;
-
-        LOG.info( "External Cassandra resource at {} is ready!", tempDir.toString() );
         lock.notifyAll();
     }
 
@@ -322,112 +238,69 @@ public class CassandraResource extends ExternalResource {
     @Override
     protected synchronized void after() {
         super.after();
-
-        shutdown = new Thread() {
-            @Override
-            public void run() {
-                try {
-                    Thread.currentThread();
-                    Thread.sleep( 100L );
-                }
-                catch ( InterruptedException ignored ) {
-                }
-
-
-                if ( schemaManager != null ) {
-                    LOG.info( "Destroying schemaManager..." );
-                    try {
-                        schemaManager.destroy();
-                    }
-                    catch ( Exception e ) {
-                        LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
-                    }
-
-                    LOG.info( "SchemaManager destroyed..." );
-                }
-
-                applicationContext.stop();
-                LOG.info( "ApplicationContext stopped..." );
-            }
-        };
-
-        shutdown.start();
+        //We don't want to actually remove our schema.  Since our tests are multi threaded, the resource may still be
+        // in use
+        //
+        //        shutdown = new Thread() {
+        //            @Override
+        //            public void run() {
+        //                try {
+        //                    Thread.currentThread();
+        //                    Thread.sleep( 100L );
+        //                }
+        //                catch ( InterruptedException ignored ) {
+        //                }
+        //
+        //
+        //                if ( schemaManager != null ) {
+        //                    LOG.info( "Destroying schemaManager..." );
+        //                    try {
+        //                        schemaManager.destroy();
+        //                    }
+        //                    catch ( Exception e ) {
+        //                        LOG.error( "Ignoring failures while dropping keyspaces: {}", e.getMessage() );
+        //                    }
+        //
+        //                    LOG.info( "SchemaManager destroyed..." );
+        //                }
+        //
+        //                applicationContext.stop();
+        //                LOG.info( "ApplicationContext stopped..." );
+        //            }
+        //        };
+        //
+        //        shutdown.start();
     }
 
 
     /**
      * Loads the specified {@link SchemaManager} or the default manager if the manager name that is provided is null.
      *
-     * @param schemaManagerName the name of the SchemaManager to load, or null
+     *
      */
-    private void loadSchemaManager( String schemaManagerName ) {
+    private void loadSchemaManager() {
         if ( !applicationContext.isActive() ) {
             LOG.info( "Restarting context..." );
             applicationContext.refresh();
         }
 
-        if ( schemaManagerName != null ) {
-            LOG.info( "Looking up SchemaManager impl: {}", schemaManagerName );
-            this.schemaManager = applicationContext.getBean( schemaManagerName, SchemaManager.class );
-        }
-        else {
-            LOG.info( "The SchemaManager is not specified - using the default SchemaManager impl" );
-            this.schemaManager = applicationContext.getBean( SchemaManager.class );
-        }
+
+        LOG.info( "The SchemaManager is not specified - using the default SchemaManager impl" );
+        this.schemaManager = applicationContext.getBean( SchemaManager.class );
 
         schemaManager.create();
         schemaManager.populateBaseData();
     }
 
 
-    /**
-     * Creates a new instance of the CassandraResource with rpc and storage ports that may or may not be the default
-     * ports. If either port is taken, an alternative free port is found.
-     *
-     * @param schemaManagerName the name of the schemaManager to use
-     *
-     * @return a new CassandraResource with possibly non-default ports
-     */
-    public static CassandraResource newWithAvailablePorts( String schemaManagerName ) {
-        // Uncomment to test for Surefire Failures
-        // System.setSecurityManager( new NoExitSecurityManager( System.getSecurityManager() ) );
-
+    public static CassandraResource newWithMavenAllocatedPorts() {
         synchronized ( lock ) {
-            if ( instance != null ) {
-                return instance;
-            }
 
-            int rpcPort = AvailablePortFinder
-                    .getNextAvailable( CassandraResource.DEFAULT_RPC_PORT + RandomUtils.nextInt( 1000 ) );
-            int storagePort = AvailablePortFinder
-                    .getNextAvailable( CassandraResource.DEFAULT_STORAGE_PORT + RandomUtils.nextInt( 1000 ) );
-            int sslStoragePort = AvailablePortFinder
-                    .getNextAvailable( CassandraResource.DEFAULT_SSL_STORAGE_PORT + RandomUtils.nextInt( 1000 ) );
-            int nativeTransportPort = AvailablePortFinder
-                    .getNextAvailable( CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT + RandomUtils.nextInt( 1000 ) );
-
-            if ( rpcPort == storagePort ) {
-                storagePort++;
-                storagePort = AvailablePortFinder.getNextAvailable( storagePort );
-            }
-
-            if ( sslStoragePort == storagePort ) {
-                sslStoragePort++;
-                sslStoragePort = AvailablePortFinder.getNextAvailable( sslStoragePort );
+            //don't keep re-initializing if it's already done
+            if(instance != null){
+                return instance;
             }
 
-            instance = new CassandraResource( schemaManagerName, rpcPort, storagePort, sslStoragePort,
-                    nativeTransportPort );
-
-            LOG.info( "Created a new instance of CassandraResource: {}", instance );
-            LOG.info( "Cassandra using ports {} and {}", storagePort, sslStoragePort );
-            return instance;
-        }
-    }
-
-
-    public static CassandraResource newWithMavenAllocatedPorts() {
-        synchronized ( lock ) {
             Properties props = new Properties();
             try {
                 props.load( ClassLoader.getSystemResourceAsStream( "project.properties" ) );
@@ -444,10 +317,6 @@ public class CassandraResource extends ExternalResource {
             int nativeTransportPort = Integer.parseInt( props.getProperty( "cassandra.nativeTransportPort",
                     Integer.toString( CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT ) ) );
             String host = props.getProperty( "cassandra.host", DEFAULT_HOST );
-            //          int rpcPort = CassandraResource.DEFAULT_RPC_PORT;
-            //          int storagePort = CassandraResource.DEFAULT_STORAGE_PORT ;
-            //          int sslStoragePort = CassandraResource.DEFAULT_SSL_STORAGE_PORT;
-            //          int nativeTransportPort = CassandraResource.DEFAULT_NATIVE_TRANSPORT_PORT;
 
             System.setProperty( "cassandra.url", host + ":" + Integer.toString( rpcPort ) );
             System.setProperty( "cassandra.cluster", props.getProperty( "cassandra.cluster", "Usergrid" ) );
@@ -466,7 +335,7 @@ public class CassandraResource extends ExternalResource {
                     new Object[] { rpcPort, storagePort, sslStoragePort, nativeTransportPort } );
 
 
-            instance = new CassandraResource( null, rpcPort, storagePort, sslStoragePort, nativeTransportPort );
+            instance = new CassandraResource( rpcPort, storagePort, sslStoragePort, nativeTransportPort );
 
             LOG.info( "Created a new instance of CassandraResource: {}", instance );
             LOG.info( "Cassandra using ports {} and {}", storagePort, sslStoragePort );
@@ -487,45 +356,6 @@ public class CassandraResource extends ExternalResource {
     }
 
 
-    /**
-     * Uses a project.properties file that Maven does substitution on to to replace the value of a property with the
-     * path to the Maven build directory (a.k.a. target). It then uses this path to generate a random String which it
-     * uses to append a path component to so a unique directory is selected. If already present it's deleted, then the
-     * directory is created.
-     *
-     * @return a unique temporary directory
-     *
-     * @throws IOException if we cannot access the properties file
-     */
-    public static File getTempDirectory() throws IOException {
-        File tmpdir;
-        File basedir = new File( ( String ) getProjectProperties().get( TARGET_DIRECTORY_KEY ) );
-        String comp = RandomStringUtils.randomAlphanumeric( 7 );
-        tmpdir = new File( basedir, comp );
-
-        if ( tmpdir.exists() ) {
-            LOG.info( "Deleting directory: {}", tmpdir );
-            FileUtils.forceDelete( tmpdir );
-        }
-        else {
-            LOG.info( "Creating temporary directory: {}", tmpdir );
-            FileUtils.forceMkdir( tmpdir );
-        }
-
-        return tmpdir;
-    }
-
-
-    public static String getTargetDir() throws IOException {
-        File basedir = new File( ( String ) getProjectProperties().get( TARGET_DIRECTORY_KEY ) );
-        return basedir.getAbsolutePath();
-    }
-
-
-    public static String getJammArgument() throws IOException {
-        return ( String ) getProjectProperties().get( JAMM_PATH );
-    }
-
 
     public static Properties getProjectProperties() throws IOException {
         if ( properties == null ) {
@@ -534,4 +364,18 @@ public class CassandraResource extends ExternalResource {
         }
         return properties;
     }
+
+
+    @Override
+    public String toString() {
+        return "CassandraResource{" +
+                "initialized=" + initialized +
+                ", rpcPort=" + rpcPort +
+                ", storagePort=" + storagePort +
+                ", sslStoragePort=" + sslStoragePort +
+                ", nativeTransportPort=" + nativeTransportPort +
+                ", applicationContext=" + applicationContext +
+                ", schemaManager=" + schemaManager +
+                "} " + super.toString();
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/500a3276/stack/test-utils/src/test/java/org/apache/usergrid/cassandra/CassandraResourceTest.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/test/java/org/apache/usergrid/cassandra/CassandraResourceTest.java b/stack/test-utils/src/test/java/org/apache/usergrid/cassandra/CassandraResourceTest.java
index 50f4d35..bbecfbc 100644
--- a/stack/test-utils/src/test/java/org/apache/usergrid/cassandra/CassandraResourceTest.java
+++ b/stack/test-utils/src/test/java/org/apache/usergrid/cassandra/CassandraResourceTest.java
@@ -80,12 +80,7 @@ public class CassandraResourceTest {
     }
 
 
-    @Test
-    public void testTmpDirectory() throws Exception {
-        File tmpdir = CassandraResource.getTempDirectory();
-        assertTrue( tmpdir.toString().contains( "target" ) );
-        assertTrue( tmpdir.exists() );
-    }
+
 
 
     /**


[07/27] incubator-usergrid git commit: Committing tests, may be some bugs still

Posted by to...@apache.org.
Committing tests, may be some bugs still


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8bab4e71
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8bab4e71
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8bab4e71

Branch: refs/heads/two-dot-o
Commit: 8bab4e71aab2e8d9dcc85ccbe0b9e204c1c9dcc8
Parents: d4ecbe1
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 7 09:44:35 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 7 09:44:35 2014 -0700

----------------------------------------------------------------------
 .../index/impl/ElasticSearchResource.java       | 113 ++++++++++---------
 .../java/org/apache/usergrid/rest/ITSetup.java  |  16 +--
 .../rest/management/RegistrationIT.java         |  61 +++++-----
 .../usergrid/cassandra/CassandraResource.java   |  18 +--
 4 files changed, 108 insertions(+), 100 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8bab4e71/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
index 4145e32..661547a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
@@ -92,62 +92,65 @@ public class ElasticSearchResource extends EnvironResource {
 
 
     public synchronized ElasticSearchResource startEs(){
-        if ( node != null ) {
-            return this;
-        }
-
-
-        //override the system properties in the Archiaus env
-        port = AvailablePortFinder.getNextAvailable( 9300 );
-
-        final String host = "127.0.0.1";
-        System.setProperty( IndexFig.ELASTICSEARCH_HOSTS, host );
-        System.setProperty( IndexFig.ELASTICSEARCH_PORT, port + "" );
-
-        //we have to create this AFTER we set our system properties, or they won't get picked upt
-        Injector injector = Guice.createInjector( new GuicyFigModule( IndexFig.class ) );
-        IndexFig indexFig = injector.getInstance( IndexFig.class );
-
-
-        final String clusterName = indexFig.getClusterName();
-
-        File tempDir;
-        try {
-            tempDir = getTempDirectory();
-        }
-        catch ( Exception ex ) {
-            throw new RuntimeException( "Fatal error unable to create temp dir, start embedded ElasticSearch", ex );
-        }
-
-
-        Settings settings = ImmutableSettings.settingsBuilder()
-                .put("cluster.name", clusterName)
-                .put("network.publish_host", host)
-                .put("transport.tcp.port", port)
-                .put("discovery.zen.ping.multicast.enabled", "false")
-                .put("node.http.enabled", false)
-                .put("path.logs", tempDir.toString())
-                .put("path.data", tempDir.toString())
-                .put("index.store.type", "default")
-                .put("index.number_of_shards", 1)
-                .put("index.number_of_replicas", 1)
-                .build();
-
-
-        log.info( "-----------------------------------------------------------------------" );
-        log.info( "Starting ElasticSearch embedded server settings: \n" + settings.getAsMap() );
-        log.info( "-----------------------------------------------------------------------" );
-
-
-        node = NodeBuilder.nodeBuilder().settings( settings ).clusterName( indexFig.getClusterName() ).client( false ).data( true ).build().start();
-
-        Runtime.getRuntime().addShutdownHook( new Thread() {
-            @Override
-            public void run() {
-                shutdown();
-            }
-        } );
 
+//
+//        if ( node != null ) {
+//            return this;
+//        }
+//
+//
+//        //override the system properties in the Archiaus env
+//        port = AvailablePortFinder.getNextAvailable( 9300 );
+//
+//        final String host = "127.0.0.1";
+//        System.setProperty( IndexFig.ELASTICSEARCH_HOSTS, host );
+//        System.setProperty( IndexFig.ELASTICSEARCH_PORT, port + "" );
+//
+//        //we have to create this AFTER we set our system properties, or they won't get picked upt
+//        Injector injector = Guice.createInjector( new GuicyFigModule( IndexFig.class ) );
+//        IndexFig indexFig = injector.getInstance( IndexFig.class );
+//
+//
+//        final String clusterName = indexFig.getClusterName();
+//
+//        File tempDir;
+//        try {
+//            tempDir = getTempDirectory();
+//        }
+//        catch ( Exception ex ) {
+//            throw new RuntimeException( "Fatal error unable to create temp dir, start embedded ElasticSearch", ex );
+//        }
+//
+//
+//        Settings settings = ImmutableSettings.settingsBuilder()
+//                .put("cluster.name", clusterName)
+//                .put("network.publish_host", host)
+//                .put("transport.tcp.port", port)
+//                .put("discovery.zen.ping.multicast.enabled", "false")
+//                .put("node.http.enabled", false)
+//                .put("path.logs", tempDir.toString())
+//                .put("path.data", tempDir.toString())
+//                .put("index.store.type", "default")
+//                .put("index.number_of_shards", 1)
+//                .put("index.number_of_replicas", 1)
+//                .build();
+//
+//
+//        log.info( "-----------------------------------------------------------------------" );
+//        log.info( "Starting ElasticSearch embedded server settings: \n" + settings.getAsMap() );
+//        log.info( "-----------------------------------------------------------------------" );
+//
+//
+//        node = NodeBuilder.nodeBuilder().settings( settings ).clusterName( indexFig.getClusterName() ).client( false ).data( true ).build().start();
+//
+//        Runtime.getRuntime().addShutdownHook( new Thread() {
+//            @Override
+//            public void run() {
+//                shutdown();
+//            }
+//        } );
+//
+//        return this;
         return this;
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8bab4e71/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java b/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
index 0d6a957..112899e 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/ITSetup.java
@@ -55,7 +55,7 @@ public class ITSetup extends ExternalResource {
     private SignInProviderFactory providerFactory;
     private Properties properties;
 
-    private boolean setupCalled = false;
+//    private boolean setupCalled = false;
     private boolean ready = false;
     private URI uri;
 
@@ -78,7 +78,9 @@ public class ITSetup extends ExternalResource {
         tomcatResource = TomcatResource.instance;
         tomcatResource.setWebAppsPath( "src/main/webapp" );
 
-        elasticSearchResource = new ElasticSearchResource().startEs();
+//        elasticSearchResource = new ElasticSearchResource().startEs();
+
+        elasticSearchResource = new ElasticSearchResource();
 
     }
 
@@ -100,12 +102,12 @@ public class ITSetup extends ExternalResource {
             tokenService =       cassandraResource.getBean( TokenService.class );
             providerFactory =    cassandraResource.getBean( SignInProviderFactory.class );
             applicationCreator = cassandraResource.getBean( ApplicationCreator.class );
-            managementService =  cassandraResource.getBean( ManagementService.class );
+//            managementService =  cassandraResource.getBean( ManagementService.class );
 
-            if ( !setupCalled ) {
-                managementService.setup();
-                setupCalled = true;
-            }
+//            if ( !setupCalled ) {
+//                managementService.setup();
+//                setupCalled = true;
+//            }
 
             String esStartup = properties.getProperty("elasticsearch.startup");
             if ( "embedded".equals(esStartup)) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8bab4e71/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java
----------------------------------------------------------------------
diff --git a/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java b/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java
index 580be03..aca33ba 100644
--- a/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java
+++ b/stack/rest/src/test/java/org/apache/usergrid/rest/management/RegistrationIT.java
@@ -32,13 +32,20 @@ import javax.mail.internet.MimeMultipart;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.MultivaluedMap;
 
+import com.eaio.uuid.UUIDGen;
 import com.fasterxml.jackson.databind.JsonNode;
+
+import org.junit.Rule;
 import org.junit.Test;
 import org.jvnet.mock_javamail.Mailbox;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.apache.usergrid.management.UserInfo;
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 import org.apache.usergrid.rest.AbstractRestIT;
+import org.apache.usergrid.rest.TestContextSetup;
+import org.apache.usergrid.rest.test.security.TestAppUser;
+import org.apache.usergrid.rest.test.security.TestUser;
 
 import org.apache.commons.lang.StringUtils;
 
@@ -64,6 +71,9 @@ public class RegistrationIT extends AbstractRestIT {
 
     private static final Logger logger = LoggerFactory.getLogger( RegistrationIT.class );
 
+    @Rule
+    public TestContextSetup context = new TestContextSetup( this );
+
 
     @Test
     public void postCreateOrgAndAdmin() throws Exception {
@@ -76,22 +86,32 @@ public class RegistrationIT extends AbstractRestIT {
             setTestProperty( PROPERTIES_ADMIN_USERS_REQUIRE_CONFIRMATION, "false" );
             setTestProperty( PROPERTIES_SYSADMIN_EMAIL, "sysadmin-1@mockserver.com" );
 
-            JsonNode node = postCreateOrgAndAdmin( "test-org-1", "test-user-1", "Test User", 
-                    "test-user-1@mockserver.com", "testpassword" );
+//            JsonNode node = postCreateOrgAndAdmin( "test-org-1", "test-user-1", "Test User",
+//                    "test-user-1@mockserver.com", "testpassword" );
 
-            if (true ) return;
-            
-            refreshIndex("test-organization", "test-app");
 
-            UUID owner_uuid =
-                    UUID.fromString( node.findPath( "data" ).findPath( "owner" ).findPath( "uuid" ).textValue() );
+            final String username = "registrationUser"+UUIDGenerator.newTimeUUID();
+            final String email = username+"@usergrid.com" ;
+            final String password = "password";
+
+            final TestUser user1 = new TestAppUser(username , password, email);
+
+            context.withOrg( "org" + UUIDGenerator.newTimeUUID() ).withApp( "app" + UUIDGenerator.newTimeUUID() ).withUser( user1 ).createNewOrgAndUser();
+            context.createAppForOrg();
+
+            final UUID owner_uuid = context.getActiveUser().getUuid();
+            
+//            refreshIndex("test-organization", "test-app");
+//
+//            UUID owner_uuid =
+//                    UUID.fromString( node.findPath( "data" ).findPath( "owner" ).findPath( "uuid" ).textValue() );
 
             List<Message> inbox = org.jvnet.mock_javamail.Mailbox.get( "test-user-1@mockserver.com" );
 
             assertFalse( inbox.isEmpty() );
 
             Message account_confirmation_message = inbox.get( 0 );
-            assertEquals( "User Account Confirmation: test-user-1@mockserver.com",
+            assertEquals( "User Account Confirmation: " + email,
                     account_confirmation_message.getSubject() );
 
             String token = getTokenFromMessage( account_confirmation_message );
@@ -99,11 +119,11 @@ public class RegistrationIT extends AbstractRestIT {
 
             setup.getMgmtSvc().disableAdminUser( owner_uuid );
 
-            refreshIndex("test-organization", "test-app");
+            refreshIndex(context.getOrgName(), context.getAppName());
 
             try {
                 resource().path( "/management/token" ).queryParam( "grant_type", "password" )
-                        .queryParam( "username", "test-user-1" ).queryParam( "password", "testpassword" )
+                        .queryParam( "username", username ).queryParam( "password", password )
                         .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
                         .get( String.class );
                 fail( "request for disabled user should fail" );
@@ -117,7 +137,7 @@ public class RegistrationIT extends AbstractRestIT {
             setup.getMgmtSvc().deactivateUser( setup.getEmf().getManagementAppId(), owner_uuid );
             try {
                 resource().path( "/management/token" ).queryParam( "grant_type", "password" )
-                        .queryParam( "username", "test-user-1" ).queryParam( "password", "testpassword" )
+                        .queryParam( "username", username ).queryParam( "password", password)
                         .accept( MediaType.APPLICATION_JSON ).type( MediaType.APPLICATION_JSON_TYPE )
                         .get( String.class );
                 fail( "request for deactivated user should fail" );
@@ -156,25 +176,6 @@ public class RegistrationIT extends AbstractRestIT {
     }
 
 
-    public JsonNode postCreateOrgAndAdmin( String organizationName, String username, String name, 
-            String email, String password ) throws IOException {
-
-        JsonNode node = null;
-        Map<String, String> payload = hashMap( "email", email )
-                .map( "username", username )
-                .map( "name", name ).map( "password", password )
-                .map( "organization", organizationName );
-
-        node = mapper.readTree( resource().path( "/management/organizations" )
-                .accept( MediaType.APPLICATION_JSON )
-                .type( MediaType.APPLICATION_JSON_TYPE )
-                .post( String.class, payload ));
-
-        assertNotNull( node );
-        logNode( node );
-        return node;
-    }
-
 
     @SuppressWarnings({ "unchecked", "rawtypes" })
     public JsonNode postAddAdminToOrg( String organizationName, String email, String password, String token ) throws IOException {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8bab4e71/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
----------------------------------------------------------------------
diff --git a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
index 5643710..401e026 100644
--- a/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
+++ b/stack/test-utils/src/main/java/org/apache/usergrid/cassandra/CassandraResource.java
@@ -58,6 +58,8 @@ import org.apache.commons.lang.math.RandomUtils;
  * The following property expansion macro should be placed in this project.properties file:
  * <p/>
  * target.directory=${pom.build.directory}
+ *
+ * TODO this class does 2 things.  It loads spring and starts cassandra.  We should separate these concerns.
  */
 public class CassandraResource extends ExternalResource {
     public static final Logger LOG = LoggerFactory.getLogger( CassandraResource.class );
@@ -293,14 +295,14 @@ public class CassandraResource extends ExternalResource {
                 return;
             }
             
-            if ( forkCassandra ) {
-                startCassandraForked();
-            } else if (externalCassandra) {
-              startCassandraExternal();
-            }else {
-              
-                startCassandraEmbedded();
-            }
+//            if ( forkCassandra ) {
+//                startCassandraForked();
+//            } else if (externalCassandra) {
+//              startCassandraExternal();
+//            }else {
+//
+//                startCassandraEmbedded();
+//            }
         }
     }
     private void addShutdownHook(){


[25/27] incubator-usergrid git commit: Ignoring tests until we re-visit the scheduler.

Posted by to...@apache.org.
Ignoring tests until we re-visit the scheduler.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e7b77581
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e7b77581
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e7b77581

Branch: refs/heads/two-dot-o
Commit: e7b77581ad254ca841a319812500649003db5deb
Parents: 75123da
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 12 12:10:30 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 12 12:10:30 2014 -0700

----------------------------------------------------------------------
 .../org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java  | 1 -
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime2IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime3IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime4IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime5IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime6IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime7IT.java    | 2 ++
 .../java/org/apache/usergrid/batch/job/SchedulerRuntime8IT.java    | 2 +-
 9 files changed, 15 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
index abf54cb..f939d0d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
@@ -36,7 +36,6 @@ import org.junit.Ignore;
 /**
  * Class to test job runtimes
  */
-@Ignore("These tests no longer work with schared spring context. Need to re-evaluate")
 public class AbstractSchedulerRuntimeIT {
 
     protected static final int DEFAULT_COUNT = 10;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java
index d0e43cc..887d868 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime1IT.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTrue;
 
 import java.util.concurrent.TimeUnit;
 
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -33,6 +34,7 @@ import org.apache.usergrid.persistence.entities.JobData;
  * Class to test job runtimes.
  */
 @Concurrent
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime1IT extends AbstractSchedulerRuntimeIT {
 	private static final Logger logger = LoggerFactory.getLogger(SchedulerRuntime1IT.class.getName());
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime2IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime2IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime2IT.java
index fb66427..0f9fb75 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime2IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime2IT.java
@@ -21,6 +21,7 @@ package org.apache.usergrid.batch.job;
 import java.util.concurrent.TimeUnit;
 import static org.apache.usergrid.batch.job.AbstractSchedulerRuntimeIT.cassandraResource;
 
+import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -34,6 +35,7 @@ import static org.junit.Assert.assertTrue;
  * Class to test job runtimes
  */
 @Concurrent
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime2IT extends AbstractSchedulerRuntimeIT {
 	private static final Logger logger = LoggerFactory.getLogger(SchedulerRuntime2IT.class.getName());
     /** Test the scheduler ramps up correctly when there are more jobs to be read after a pause */

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime3IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime3IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime3IT.java
index 2968eda..3002377 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime3IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime3IT.java
@@ -23,6 +23,7 @@ import org.apache.usergrid.cassandra.Concurrent;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.apache.usergrid.persistence.entities.JobStat;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -33,6 +34,7 @@ import static org.junit.Assert.assertTrue;
  * Class to test job runtimes
  */
 @Concurrent
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime3IT extends AbstractSchedulerRuntimeIT {
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime4IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime4IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime4IT.java
index 5ad56a5..7b97db4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime4IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime4IT.java
@@ -21,6 +21,7 @@ import org.apache.usergrid.cassandra.Concurrent;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.apache.usergrid.persistence.entities.JobStat;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -31,6 +32,7 @@ import static org.junit.Assert.assertTrue;
  * Class to test job runtimes
  */
 @Concurrent
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime4IT extends AbstractSchedulerRuntimeIT {
     /**
      * Test the scheduler ramps up correctly when there are more jobs to be read after a pause when the job specifies

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime5IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime5IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime5IT.java
index 1db35b8..eed8490 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime5IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime5IT.java
@@ -21,6 +21,7 @@ import org.apache.usergrid.cassandra.Concurrent;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.apache.usergrid.persistence.entities.JobStat;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -31,6 +32,7 @@ import static org.junit.Assert.assertTrue;
  * Class to test job runtimes
  */
 @Concurrent
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime5IT extends AbstractSchedulerRuntimeIT {
     /**
      * Test the scheduler ramps up correctly when there are more jobs to be read after a pause when the job specifies

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime6IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime6IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime6IT.java
index 4c74f20..4b9b232 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime6IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime6IT.java
@@ -23,6 +23,7 @@ import org.apache.usergrid.cassandra.Concurrent;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.apache.usergrid.persistence.entities.JobStat;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -35,6 +36,7 @@ import static org.junit.Assert.assertTrue;
  * Class to test job runtimes
  */
 @Concurrent
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime6IT extends AbstractSchedulerRuntimeIT {
 
     /**

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime7IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime7IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime7IT.java
index c133f1a..b23cf78 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime7IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime7IT.java
@@ -21,6 +21,7 @@ import org.apache.usergrid.cassandra.Concurrent;
 import org.apache.usergrid.persistence.entities.JobData;
 import org.apache.usergrid.persistence.entities.JobStat;
 
+import org.junit.Ignore;
 import org.junit.Test;
 
 import static org.junit.Assert.assertEquals;
@@ -32,6 +33,7 @@ import static org.junit.Assert.assertTrue;
  */
 @Concurrent
 //@org.junit.Ignore( "Todd you need to take a look at this since it's not clear to me what was intended in this test." )
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime7IT extends AbstractSchedulerRuntimeIT {
 
     /** Test that we're only running once, even when a job exceeds the heartbeat time */

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e7b77581/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime8IT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime8IT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime8IT.java
index 5b9271b..0eabace 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime8IT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/SchedulerRuntime8IT.java
@@ -36,7 +36,7 @@ import static org.junit.Assert.assertFalse;
  * Class to test job runtimes
  */
 @Concurrent
-@Ignore("Ignoring until we can diagnose query issue")
+@Ignore("These tests no longer work with shared spring context. Need to re-evaluate")
 public class SchedulerRuntime8IT extends AbstractSchedulerRuntimeIT {
     
     /**


[14/27] incubator-usergrid git commit: Finished CP cleanup

Posted by to...@apache.org.
Finished CP cleanup


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/1ea42f83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/1ea42f83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/1ea42f83

Branch: refs/heads/two-dot-o
Commit: 1ea42f834eadd99b545f6ae4e02b614f20afe936
Parents: 6ae1bef
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 16:20:41 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 16:20:41 2014 -0700

----------------------------------------------------------------------
 .../apache/usergrid/corepersistence/CpRelationManager.java    | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/1ea42f83/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 20309a7..561c269 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -619,8 +619,8 @@ public class CpRelationManager implements RelationManager {
     }
 
 
-    public Entity addToCollection( String collName, EntityRef itemRef,
-            org.apache.usergrid.persistence.model.entity.Entity memberEntity, boolean connectBack ) 
+    public Entity addToCollection(final String collName, final EntityRef itemRef,
+            final org.apache.usergrid.persistence.model.entity.Entity memberEntity, final boolean connectBack )
         throws Exception {
 
         // don't fetch entity if we've already got one
@@ -646,12 +646,9 @@ public class CpRelationManager implements RelationManager {
                 applicationScope.getApplication(),
                 applicationScope.getApplication(),
                 CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
-        EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager( memberScope );
 
         //TODO, this double load should disappear once events are in
         Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
-        org.apache.usergrid.persistence.model.entity.Entity memberEntity =
-            ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId));
 
         if ( memberEntity == null ) {
             throw new RuntimeException(


[02/27] incubator-usergrid git commit: Updates constants.

Posted by to...@apache.org.
Updates constants.

Fixes scope in query

Need to finish version query


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ef77a5a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ef77a5a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ef77a5a4

Branch: refs/heads/two-dot-o
Commit: ef77a5a4eeca44496bf0ced75cf662f49040cba8
Parents: 2db132c
Author: Todd Nine <tn...@apigee.com>
Authored: Thu Nov 6 16:55:13 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Thu Nov 6 17:43:18 2014 -0700

----------------------------------------------------------------------
 .../index/impl/EsEntityIndexBatchImpl.java      |  11 +-
 .../index/impl/EsEntityIndexImpl.java           | 126 ++-
 .../persistence/index/impl/EsQueryVistor.java   |   3 +-
 .../persistence/index/impl/IndexingUtils.java   | 116 +--
 .../usergrid/persistence/index/query/Query.java |  28 +-
 .../impl/EntityConnectionIndexImplTest.java     |  45 +-
 .../persistence/index/impl/EntityIndexTest.java | 785 ++++++++++---------
 .../persistence/index/impl/EsRunner.java        |   2 +-
 .../persistence/query/tree/GrammarTreeTest.java |   2 +-
 9 files changed, 590 insertions(+), 528 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 5a19d1e..01f22c0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -61,8 +61,8 @@ import com.google.common.base.Joiner;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT_FIELDNAME;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
@@ -116,7 +116,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
 
         if ( log.isDebugEnabled() ) {
             log.debug( "Indexing entity {}:{} in scope\n   app {}\n   "
-                    + "owner {}\n   name {}\n   type {} \n scope type{}", new Object[] {
+                    + "owner {}\n   name {}\n   type {} \n scope type {}", new Object[] {
                     entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
                     indexScope.getOwner(), indexScope.getName(), entityType, context
             } );
@@ -255,10 +255,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
         final Map entityMap = entityToMap( entity );
 
         //add the context for filtering later
-        entityMap.put( ENTITY_CONTEXT, context );
+        entityMap.put( ENTITY_CONTEXT_FIELDNAME, context );
 
         //but the fieldname
-        entityMap.put( ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() );
+        //we have to prefix because we use query equality to seek this later.  TODO see if we can make this more declarative
+        entityMap.put( STRING_PREFIX+ ENTITYID_ID_FIELDNAME, IndexingUtils.idString(entity.getId()).toLowerCase() );
 
         return entityMap;
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 7378ecd..d7955c4 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -33,17 +33,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
 import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.search.SearchScrollRequestBuilder;
 import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.client.Client;
 import org.elasticsearch.common.settings.ImmutableSettings;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermFilterBuilder;
 import org.elasticsearch.indices.IndexAlreadyExistsException;
 import org.elasticsearch.indices.IndexMissingException;
 import org.elasticsearch.search.SearchHit;
@@ -54,8 +51,6 @@ import org.elasticsearch.search.sort.SortOrder;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.lucene.queryparser.xml.FilterBuilderFactory;
-
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.util.Health;
 import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -76,9 +71,9 @@ import com.google.inject.Inject;
 import com.google.inject.assistedinject.Assisted;
 
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.DOC_ID_SEPARATOR_SPLITTER;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_ID_FIELDNAME;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.SPLITTER;
 import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
 
 
@@ -148,12 +143,12 @@ public class EsEntityIndexImpl implements EntityIndex {
             final int numberOfShards = config.getNumberOfShards();
             final int numberOfReplicas = config.getNumberOfReplicas();
 
-            Settings settings = ImmutableSettings.settingsBuilder()
-            .put("index.number_of_shards", numberOfShards )
-            .put("index.number_of_replicas", numberOfReplicas).build();
+            Settings settings = ImmutableSettings.settingsBuilder().put( "index.number_of_shards", numberOfShards )
+                                                 .put( "index.number_of_replicas", numberOfReplicas ).build();
 
 
-            final  CreateIndexResponse cir = admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
+            final CreateIndexResponse cir =
+                    admin.indices().prepareCreate( indexName ).setSettings( settings ).execute().actionGet();
 
             logger.info( "Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged() );
 
@@ -175,8 +170,8 @@ public class EsEntityIndexImpl implements EntityIndex {
 
 
     /**
-     * Tests writing a document to a new index to ensure it's working correctly. 
-     * See this post: http://s.apache.org/index-missing-exception
+     * Tests writing a document to a new index to ensure it's working correctly. See this post:
+     * http://s.apache.org/index-missing-exception
      */
     private void testNewIndex() {
 
@@ -188,18 +183,17 @@ public class EsEntityIndexImpl implements EntityIndex {
             public boolean doOp() {
                 final String tempId = UUIDGenerator.newTimeUUID().toString();
 
-                esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId )
-                        .setSource( DEFAULT_PAYLOAD ).get();
+                esProvider.getClient().prepareIndex( indexName, VERIFY_TYPE, tempId ).setSource( DEFAULT_PAYLOAD )
+                          .get();
 
-                logger.info( "Successfully created new document with docId {} in index {} and type {}", 
-                        tempId, indexName, VERIFY_TYPE );
+                logger.info( "Successfully created new document with docId {} in index {} and type {}", tempId,
+                        indexName, VERIFY_TYPE );
 
                 // delete all types, this way if we miss one it will get cleaned up
                 esProvider.getClient().prepareDeleteByQuery( indexName ).setTypes( VERIFY_TYPE )
-                        .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
+                          .setQuery( MATCH_ALL_QUERY_BUILDER ).get();
 
-                logger.info( "Successfully deleted all documents in index {} and type {}", 
-                        indexName, VERIFY_TYPE );
+                logger.info( "Successfully deleted all documents in index {} and type {}", indexName, VERIFY_TYPE );
 
                 return true;
             }
@@ -218,10 +212,11 @@ public class EsEntityIndexImpl implements EntityIndex {
         XContentBuilder xcb =
                 IndexingUtils.createDoubleStringIndexMapping( XContentFactory.jsonBuilder(), "_default_" );
 
-        PutIndexTemplateResponse pitr = esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" )
-                                              .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_",
+        PutIndexTemplateResponse pitr =
+                esProvider.getClient().admin().indices().preparePutTemplate( "usergrid_template" )
+                          .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_",
                         xcb ) // set mapping as the default for all types
-                .execute().actionGet();
+                        .execute().actionGet();
     }
 
 
@@ -237,40 +232,40 @@ public class EsEntityIndexImpl implements EntityIndex {
         final String context = IndexingUtils.createContextName( indexScope );
         final String[] entityTypes = searchTypes.getTypeNames();
 
-        QueryBuilder qb = query.createQueryBuilder();
+        QueryBuilder qb = query.createQueryBuilder(context);
 
-        if ( logger.isDebugEnabled() ) {
-            logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
-                    this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
-            } );
-        }
 
         SearchResponse searchResponse;
 
         if ( query.getCursor() == null ) {
-            SearchRequestBuilder srb =
-                    esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes ).setScroll(
-                            cursorTimeout + "m" )
-                          .setQuery( qb );
+            SearchRequestBuilder srb = esProvider.getClient().prepareSearch( indexName ).setTypes( entityTypes )
+                                                 .setScroll( cursorTimeout + "m" ).setQuery( qb );;
 
-            final TermFilterBuilder contextFilter = FilterBuilders.termFilter( IndexingUtils.ENTITY_CONTEXT, context );
 
-            final FilterBuilder fb = query.createFilterBuilder();
+            //we must add a "must" operation to our entity context.
+
 
+            //set our final query
+
+
+            if ( logger.isDebugEnabled() ) {
+                        logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
+                                this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
+                        } );
+                    }
+
+
+
+            final FilterBuilder fb = query.createFilterBuilder();
 
 
             //we have post filters, apply them
             if ( fb != null ) {
-//                final FilterBuilder postFilters = FilterBuilders.andFilter(fb, contextFilter  );
-
                 logger.debug( "   Filter: {} ", fb.toString() );
-//                srb = srb.setPostFilter( postFilters );
-            }
-            //no other post filters, just the types
-            else{
-//                srb.setPostFilter( contextFilter );
+                srb = srb.setPostFilter( fb );
             }
 
+
             srb = srb.setFrom( 0 ).setSize( query.getLimit() );
 
             for ( Query.SortPredicate sp : query.getSortPredicates() ) {
@@ -289,20 +284,20 @@ public class EsEntityIndexImpl implements EntityIndex {
                 // to ignore any fields that are not present.
 
                 final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName )
-                        .order( order ).ignoreUnmapped( true );
+                final FieldSortBuilder stringSort =
+                        SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( stringSort );
                 logger.debug( "   Sort: {} order by {}", stringFieldName, order.toString() );
 
                 final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName )
-                        .order( order ).ignoreUnmapped( true );
+                final FieldSortBuilder numberSort =
+                        SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( numberSort );
                 logger.debug( "   Sort: {} order by {}", numberFieldName, order.toString() );
 
                 final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
-                final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
-                        .order( order ).ignoreUnmapped( true );
+                final FieldSortBuilder booleanSort =
+                        SortBuilders.fieldSort( booleanFieldName ).order( order ).ignoreUnmapped( true );
                 srb.addSort( booleanSort );
                 logger.debug( "   Sort: {} order by {}", booleanFieldName, order.toString() );
             }
@@ -329,7 +324,8 @@ public class EsEntityIndexImpl implements EntityIndex {
             }
             logger.debug( "Executing query with cursor: {} ", scrollId );
 
-            SearchScrollRequestBuilder ssrb = esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
+            SearchScrollRequestBuilder ssrb =
+                    esProvider.getClient().prepareSearchScroll( scrollId ).setScroll( cursorTimeout + "m" );
 
             try {
                 searchResponse = ssrb.execute().actionGet();
@@ -351,7 +347,7 @@ public class EsEntityIndexImpl implements EntityIndex {
 
         for ( SearchHit hit : hits.getHits() ) {
 
-            String[] idparts = hit.getId().split( DOC_ID_SEPARATOR_SPLITTER );
+            String[] idparts = hit.getId().split( SPLITTER );
             String id = idparts[0];
             String type = idparts[1];
             String version = idparts[2];
@@ -386,7 +382,7 @@ public class EsEntityIndexImpl implements EntityIndex {
                     return true;
                 }
                 catch ( IndexMissingException e ) {
-                    logger.error( "Unable to refresh index after create. Waiting before sleeping.", e);
+                    logger.error( "Unable to refresh index after create. Waiting before sleeping.", e );
                     throw e;
                 }
             }
@@ -401,7 +397,7 @@ public class EsEntityIndexImpl implements EntityIndex {
     @Override
     public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
         Query query = new Query();
-        query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
+        query.addEqualityFilter( ENTITYID_ID_FIELDNAME, IndexingUtils.idString( id ).toLowerCase() );
         CandidateResults results = search( scope, SearchTypes.fromTypes( id.getType() ), query );
         return results;
     }
@@ -424,7 +420,6 @@ public class EsEntityIndexImpl implements EntityIndex {
 
     /**
      * Do the retry operation
-     * @param operation
      */
     private void doInRetry( final RetryOperation operation ) {
         for ( int i = 0; i < MAX_WAITS; i++ ) {
@@ -456,13 +451,13 @@ public class EsEntityIndexImpl implements EntityIndex {
     public Health getClusterHealth() {
 
         try {
-            ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
-                .health( new ClusterHealthRequest() ).get();
+            ClusterHealthResponse chr =
+                    esProvider.getClient().admin().cluster().health( new ClusterHealthRequest() ).get();
             return Health.valueOf( chr.getStatus().name() );
-        } 
-        catch (Exception ex) {
-            logger.error("Error connecting to ElasticSearch", ex);
-        } 
+        }
+        catch ( Exception ex ) {
+            logger.error( "Error connecting to ElasticSearch", ex );
+        }
 
         // this is bad, red alert!
         return Health.RED;
@@ -474,15 +469,16 @@ public class EsEntityIndexImpl implements EntityIndex {
      */
     @Override
     public Health getIndexHealth() {
-        
+
         try {
             ClusterHealthResponse chr = esProvider.getClient().admin().cluster()
-                .health( new ClusterHealthRequest( new String[] { indexName } ) ).get();
+                                                  .health( new ClusterHealthRequest( new String[] { indexName } ) )
+                                                  .get();
             return Health.valueOf( chr.getStatus().name() );
-        } 
-        catch (Exception ex) {
-            logger.error("Error connecting to ElasticSearch", ex);
-        } 
+        }
+        catch ( Exception ex ) {
+            logger.error( "Error connecting to ElasticSearch", ex );
+        }
 
         // this is bad, red alert!
         return Health.RED;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 0afe992..0e4b0ad 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -219,9 +219,10 @@ public class EsQueryVistor implements QueryVisitor {
             String svalue = (String)value;
 
             BoolQueryBuilder qb = QueryBuilders.boolQuery();  // let's do a boolean OR
-            qb.minimumNumberShouldMatch(1); 
+            qb.minimumNumberShouldMatch(1);
 
             // field is an entity/array that does not need a prefix on its name
+            //TODO is this right now that we've updated our doc structure?  Should this be "must" instead of should?
             qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
            
             // or field is just a string that does need a prefix

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index c8e0b86..c7592bc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -37,22 +37,21 @@ public class IndexingUtils {
     public static final String NUMBER_PREFIX = "nu_";
     public static final String BOOLEAN_PREFIX = "bu_";
 
-    public static final String ENTITYID_FIELDNAME = "entityId";
 
-    public static final String DOC_ID_SEPARATOR = "|";
-    public static final String DOC_ID_SEPARATOR_SPLITTER = "\\|";
 
-    // These are not allowed in document type names: _ . , | #
-    public static final String DOC_TYPE_SEPARATOR = "^";
-
-    public static final String INDEX_NAME_SEPARATOR = "^";
+    public static final String SPLITTER = "\\__";
 
-    public static final String ENTITY_CONTEXT = "_context";
+    // These are not allowed in document type names: _ . , | #
+    public static final String SEPARATOR = "__";
 
     /**
-     * To be used when we want to search all types within a scope
+     * Reserved UG fields.
      */
-    public static final String ALL_TYPES = "ALL";
+
+    public static final String ENTITY_CONTEXT_FIELDNAME = "ug_context";
+
+    public static final String ENTITYID_ID_FIELDNAME = "ug_entityId";
+
 
 
     /**
@@ -62,13 +61,35 @@ public class IndexingUtils {
       */
      public static String createContextName( IndexScope scope ) {
          StringBuilder sb = new StringBuilder();
-         sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR);
-         sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR);
+         idString(sb, scope.getOwner());
+         sb.append( SEPARATOR );
          sb.append( scope.getName() );
          return sb.toString();
      }
 
 
+    /**
+     * Append the id to the string
+     * @param builder
+     * @param id
+     */
+    public static final void idString(final StringBuilder builder, final Id id){
+        builder.append( id.getUuid() ).append( SEPARATOR )
+                .append(id.getType());
+    }
+
+
+    /**
+     * Turn the id into a string
+     * @param id
+     * @return
+     */
+    public static final String idString(final Id id){
+        final StringBuilder sb = new StringBuilder(  );
+        idString(sb, id);
+        return sb.toString();
+    }
+
 
     /**
      * Create the index name based on our prefix+appUUID+AppType
@@ -79,9 +100,8 @@ public class IndexingUtils {
     public static String createIndexName(
             String prefix, ApplicationScope applicationScope) {
         StringBuilder sb = new StringBuilder();
-        sb.append( prefix ).append(INDEX_NAME_SEPARATOR);
-        sb.append( applicationScope.getApplication().getUuid() ).append(INDEX_NAME_SEPARATOR);
-        sb.append( applicationScope.getApplication().getType() );
+        sb.append( prefix ).append( SEPARATOR );
+        idString( sb, applicationScope.getApplication() );
         return sb.toString();
     }
 
@@ -92,8 +112,8 @@ public class IndexingUtils {
      * @param entity
      * @return
      */
-    public static String createIndexDocId(final Entity entity, final String scopeType) {
-        return createIndexDocId(entity.getId(), entity.getVersion(), scopeType);
+    public static String createIndexDocId(final Entity entity, final String context) {
+        return createIndexDocId(entity.getId(), entity.getVersion(), context);
     }
 
 
@@ -101,14 +121,15 @@ public class IndexingUtils {
      * Create the doc Id. This is the entitie's type + uuid + version
      * @param entityId
      * @param version
+     * @para context The context it's indexed in
      * @return
      */
-    public static String createIndexDocId(final Id entityId, final UUID version, final String scopeType) {
+    public static String createIndexDocId(final Id entityId, final UUID version, final String context) {
         StringBuilder sb = new StringBuilder();
-        sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR);
-        sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR);
-        sb.append( version.toString() ).append( DOC_ID_SEPARATOR );
-        sb.append( scopeType);
+        idString(sb, entityId);
+        sb.append( SEPARATOR );
+        sb.append( version.toString() ).append( SEPARATOR );
+        sb.append( context);
         return sb.toString();
     }
 
@@ -135,25 +156,41 @@ public class IndexingUtils {
 
                     .startArray( "dynamic_templates" )
 
-                        // any string with field name that starts with sa_ gets analyzed
+                           //we need most specific mappings first since it's a stop on match algorithm
+
                         .startObject()
-                            .startObject( "template_1" )
-                                .field( "match", ANALYZED_STRING_PREFIX + "*" )
-                                .field( "match_mapping_type", "string" )
+                        .startObject( "context_template" )
+                            .field( "match", IndexingUtils.ENTITYID_ID_FIELDNAME )
+                                 .field( "match_mapping_type", "string" )
+                                        .startObject( "mapping" ).field( "type", "string" )
+                                             .field( "index", "not_analyzed" )
+                                        .endObject()
+                                 .endObject()
+                             .endObject()
+
+
+                        .startObject()
+                        .startObject( "context_template" )
+                            .field( "match", IndexingUtils.ENTITY_CONTEXT_FIELDNAME )
+                            .field( "match_mapping_type", "string" )
                                 .startObject( "mapping" ).field( "type", "string" )
-                                    .field( "index", "analyzed" )
+                                    .field( "index", "not_analyzed" ).endObject()
                                 .endObject()
-                            .endObject()
+                         .endObject()
+
+                        // any string with field name that starts with sa_ gets analyzed
+                        .startObject()
+                            .startObject( "template_1" ).field( "match", ANALYZED_STRING_PREFIX + "*" )
+                            .field( "match_mapping_type", "string" ).startObject( "mapping" ).field( "type", "string" )
+                            .field( "index", "analyzed" ).endObject().endObject()
                         .endObject()
 
-                            // all other strings are not analyzed
+                                // all other strings are not analyzed
                         .startObject()
                             .startObject( "template_2" )
-                //todo, should be string prefix, remove 2 field mapping
-                                .field( "match", "*" )
-                                .field( "match_mapping_type", "string" )
-                                .startObject( "mapping" )
-                                    .field( "type", "string" )
+                                    //todo, should be string prefix, remove 2 field mapping
+                                .field( "match", "*" ).field( "match_mapping_type", "string" ).startObject( "mapping" )
+                                .field( "type", "string" )
                                     .field( "index", "not_analyzed" )
                                 .endObject()
                             .endObject()
@@ -169,16 +206,7 @@ public class IndexingUtils {
                             .endObject()
                         .endObject()
 
-                //types for context direct string matching
-                .startObject( "context_template" )
-                        .field( "match", IndexingUtils.ENTITY_CONTEXT )
-                        .field( "match_mapping_type", "string" )
-                        .startObject( "mapping" )
-                            .field( "type", "string" )
-                            .field( "index", "not_analyzed" )
-                        .endObject()
-                    .endObject()
-                .endObject()
+
 
                     .endArray()
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 778134e..07bee44 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -43,6 +43,7 @@ import org.apache.commons.lang.StringUtils;
 import org.apache.usergrid.persistence.index.exceptions.IndexException;
 import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
 import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
 import org.apache.usergrid.persistence.index.query.tree.AndOperand;
 import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
 import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -142,10 +143,23 @@ public class Query {
     }
 
 
-    public QueryBuilder createQueryBuilder() {
+    public QueryBuilder createQueryBuilder(final String context) {
 
-        QueryBuilder queryBuilder = null;
 
+        QueryBuilder queryBuilder;
+
+
+        /**
+         * Add our filter for context to our query for fast execution.  Fast because it utilizes bitsets
+         * internally. See this post for more detail.
+         * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+         */
+
+
+
+
+
+        //we have a root operand.  Translate our AST into an ES search
         if ( getRootOperand() != null ) {
             QueryVisitor v = new EsQueryVistor();
             try {
@@ -154,11 +168,15 @@ public class Query {
             } catch ( IndexException ex ) {
                 throw new RuntimeException( "Error building ElasticSearch query", ex );
             }
-            queryBuilder = v.getQueryBuilder();
+
+            // TODO evaluate performance when it's an all query.  Do we need to put the context term first for performance?
+            queryBuilder = QueryBuilders.boolQuery().must(  v.getQueryBuilder() )
+                                        .must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
         } 
 
-		if ( queryBuilder == null ) {
-            queryBuilder = QueryBuilders.matchAllQuery();
+        //nothing was specified ensure we specify the context in the search
+		else {
+            queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
 		}
 
         return queryBuilder;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index c1c075e..d2423bb 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.guice.TestIndexModule;
+import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
 import org.apache.usergrid.persistence.model.entity.Entity;
@@ -51,6 +52,7 @@ import com.google.inject.Inject;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
 
 
 @RunWith( EsRunner.class )
@@ -59,15 +61,9 @@ public class EntityConnectionIndexImplTest extends BaseIT {
 
     private static final Logger log = LoggerFactory.getLogger( EntityConnectionIndexImplTest.class );
 
-    @ClassRule
-    public static ElasticSearchResource es = new ElasticSearchResource();
+//    @ClassRule
+//    public static ElasticSearchResource es = new ElasticSearchResource();
 
-    @ClassRule
-    public static CassandraRule cass = new CassandraRule();
-
-    @Inject
-    @Rule
-    public MigrationManagerRule migrationManagerRule;
 
     @Inject
     public EntityIndexFactory ecif;
@@ -121,7 +117,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         IndexScope searchScope = new IndexScopeImpl( personId, "likes" );
 
         //create another scope we index in, want to be sure these scopes are filtered
-        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "person" ), "likes" );
+        IndexScope otherIndexScope = new IndexScopeImpl( new SimpleId( UUIDGenerator.newTimeUUID(), "animal" ), "likes" );
 
         EntityIndex personLikesIndex = ecif.createEntityIndex( applicationScope );
 
@@ -142,6 +138,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
         batch.index( otherIndexScope, oj );
 
         batch.executeAndRefresh();
+        personLikesIndex.refresh();
 
         // now, let's search for muffins
         CandidateResults likes = personLikesIndex
@@ -167,18 +164,18 @@ public class EntityConnectionIndexImplTest extends BaseIT {
                 SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
                 Query.fromQL( "select *" ) );
         assertEquals( 3, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
-        assertEquals( oj.getId(), likes.get( 2 ).getId() );
+        assertContains( egg.getId(), likes );
+        assertContains( muffin.getId(), likes );
+        assertContains( oj.getId(), likes );
 
           //now lets search for all explicitly
         likes = personLikesIndex.search( searchScope,
                 SearchTypes.allTypes(),
                 Query.fromQL( "select *" ) );
         assertEquals( 3, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
-        assertEquals( oj.getId(), likes.get( 2 ).getId() );
+        assertContains( egg.getId(), likes );
+        assertContains( muffin.getId(), likes);
+        assertContains( oj.getId(), likes );
 
 
 
@@ -187,16 +184,26 @@ public class EntityConnectionIndexImplTest extends BaseIT {
                 SearchTypes.fromTypes( muffin.getId().getType(), egg.getId().getType(), oj.getId().getType() ),
                 Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+        assertContains( egg.getId(), likes);
+        assertContains( muffin.getId(), likes );
 
 
 
         //now search with no types, we should get only the results that match
         likes = personLikesIndex.search( searchScope, SearchTypes.allTypes(), Query.fromQL( "select * where stars = 5" ) );
         assertEquals( 2, likes.size() );
-        assertEquals( muffin.getId(), likes.get( 0 ).getId() );
-        assertEquals( egg.getId(), likes.get( 1 ).getId() );
+        assertContains( egg.getId(), likes);
+        assertContains( muffin.getId(), likes );
+    }
+
+
+    private void assertContains(final Id id, final CandidateResults results ){
+        for( CandidateResult result: results){
+            if(result.getId().equals(id)){
+                return;
+            }
+        }
 
+        fail(String.format("Could not find id %s in candidate results", id));
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 1bbb301..a1240f1 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -45,6 +45,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.guice.TestIndexModule;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
@@ -69,393 +70,403 @@ import static org.junit.Assert.assertNotEquals;
 @UseModules({ TestIndexModule.class })
 public class EntityIndexTest extends BaseIT {
 
-//    private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
-//
-//    @ClassRule
-//    public static CassandraRule cass = new CassandraRule();
-//
-//    @Rule
-//    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
-//
-//    @Inject
-//    @Rule
-//    public MigrationManagerRule migrationManagerRule;
-//
-//    @Inject
-//    public EntityIndexFactory eif;
-//
-//
-//
-//    @Test
-//    public void testIndex() throws IOException {
-//
-//        final int MAX_ENTITIES = 100;
-//
-//        Id appId = new SimpleId( "application" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
-//
-//
-//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-//        entityIndex.initializeIndex();
-//
-//        InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
-//        ObjectMapper mapper = new ObjectMapper();
-//        List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
-//
-//        int count = 0;
-//        StopWatch timer = new StopWatch();
-//        timer.start();
-//
-//        final EntityIndexBatch batch = entityIndex.createBatch();
-//
-//        for ( Object o : sampleJson ) {
-//
-//            Map<String, Object> item = ( Map<String, Object> ) o;
-//
-//            Entity entity = new Entity( indexScope.getName() );
-//            entity = EntityIndexMapUtils.fromMap( entity, item );
-//            EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-//
-//            batch.index( indexScope, entity );
-//
-//            if(count %1000 == 0){
-//                batch.execute();
-//            }
-//
-//
-//
-//            if ( count++ > MAX_ENTITIES ) {
-//                break;
-//            }
-//        }
-//
-//        batch.execute();
-//        timer.stop();
-//        log.info( "Total time to index {} entries {}ms, average {}ms/entry",
-//                new Object[] { count, timer.getTime(), timer.getTime() / count } );
-//
-//        entityIndex.refresh();
-//
-//
-//        testQueries( indexScope, entityIndex );
-//    }
-//
-//
-//    @Test
-//    public void testDeindex() {
-//
-//        Id appId = new SimpleId( "application" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope indexScope = new IndexScopeImpl( appId, "fastcars", entityType );
-//
-//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-//        entityIndex.initializeIndex();
-//
-//        Map entityMap = new HashMap() {{
-//            put( "name", "Ferrari 212 Inter" );
-//            put( "introduced", 1952 );
-//            put( "topspeed", 215 );
-//        }};
-//
-//
-//        Entity entity = EntityIndexMapUtils.fromMap( entityMap );
-//        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
-//        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
-//        entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
-//
-//        CandidateResults candidateResults = entityIndex.search(indexScope,  Query.fromQL( "name contains 'Ferrari*'" ) );
-//        assertEquals( 1, candidateResults.size() );
-//
-//        entityIndex.createBatch().deindex( indexScope, entity ).execute();
-//
-//        entityIndex.refresh();
-//
-//        candidateResults = entityIndex.search( indexScope, Query.fromQL( "name contains 'Ferrari*'" ) );
-//        assertEquals( 0, candidateResults.size() );
-//    }
-//
-//
-//    private void testQuery(final IndexScope scope, final EntityIndex entityIndex, final String queryString, final int num ) {
-//
-//        StopWatch timer = new StopWatch();
-//        timer.start();
-//        Query query = Query.fromQL( queryString );
-//        query.setLimit( 1000 );
-//        CandidateResults candidateResults = entityIndex.search( scope, query );
-//        timer.stop();
-//
-//        assertEquals( num, candidateResults.size() );
-//        log.debug( "Query time {}ms", timer.getTime() );
-//    }
-//
-//
-//    private void testQueries(final IndexScope scope, final EntityIndex entityIndex ) {
-//
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan Pierce'", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'morgan pierce'", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan'", 0 );
-//
-//        testQuery(scope,  entityIndex, "name contains 'Morgan'", 1 );
-//
-//        testQuery(scope,  entityIndex, "company > 'GeoLogix'", 64 );
-//
-//        testQuery(scope,  entityIndex, "gender = 'female'", 45 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan* '", 1 );
-//
-//        testQuery(scope,  entityIndex, "name = 'Morgan*'", 1 );
-//
-//
-//        // test a couple of array sub-property queries
-//
-//        int totalUsers = 102;
-//
-//        // nobody has a friend named Jack the Ripper
-//        testQuery(scope,  entityIndex, "friends.name = 'Jack the Ripper'", 0 );
-//
-//        // everybody doesn't have a friend named Jack the Ripper
-//        testQuery(scope,  entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
-//
-//        // one person has a friend named Shari Hahn
-//        testQuery(scope,  entityIndex, "friends.name = 'Wendy Moody'", 1 );
-//
-//        // everybody but 1 doesn't have a friend named Shari Hahh
-//        testQuery(scope,  entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
-//
-//    }
-//
-//
-//    /**
-//     * Tests that Entity-to-map and Map-to-entity round trip works.
-//     */
-//    @Test
-//    public void testEntityIndexMapUtils() throws IOException {
-//
-//        InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" );
-//        ObjectMapper mapper = new ObjectMapper();
-//        List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} );
-//
-//        for ( Object o : contacts ) {
-//
-//            Map<String, Object> map1 = ( Map<String, Object> ) o;
-//
-//            // convert map to entity
-//            Entity entity1 = EntityIndexMapUtils.fromMap( map1 );
-//
-//            // convert entity back to map
-//            Map map2 = EntityIndexMapUtils.toMap( entity1 );
-//
-//            // the two maps should be the same
-//            Map diff = Maps.difference( map1, map2 ).entriesDiffering();
-//            assertEquals( 0, diff.size() );
-//        }
-//    }
-//
-//
-//    @Test
-//    public void getEntityVersions() throws Exception {
-//
-//        Id appId = new SimpleId( "application" );
-//        Id ownerId = new SimpleId( "owner" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope indexScope = new IndexScopeImpl( ownerId, "user", entityType );
-//
-//
-//
-//        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
-//        entityIndex.initializeIndex();
-//
-//        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-//        Map<String, Object> properties = new LinkedHashMap<String, Object>();
-//        properties.put( "username", "edanuff" );
-//        properties.put( "email", "ed@anuff.com" );
-//        properties.put( "middlename", middleName );
-//
-//        Map entityMap = new HashMap() {{
-//            put( "username", "edanuff" );
-//            put( "email", "ed@anuff.com" );
-//            put( "middlename", middleName );
-//        }};
-//
-//        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-//        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
-//        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-//
-//
-//        final EntityIndexBatch batch = entityIndex.createBatch();
-//
-//        batch.index( indexScope, user );
-//
-//        user.setField( new StringField( "address1", "1782 address st" ) );
-//        batch.index( indexScope, user );
-//        user.setField( new StringField( "address2", "apt 508" ) );
-//        batch.index( indexScope,  user );
-//        user.setField( new StringField( "address3", "apt 508" ) );
-//        batch.index( indexScope,  user );
-//        batch.executeAndRefresh();
-//
-//        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId() );
-//
-//        assertEquals(1,  results.size());
-//        assertEquals( results.get( 0 ).getId(), user.getId() );
-//        assertEquals( results.get(0).getVersion(), user.getVersion());
-//    }
-//
-//
-//    @Test
-//    public void deleteVerification() throws Throwable {
-//
-//        Id appId = new SimpleId( "application" );
-//        Id ownerId = new SimpleId( "owner" );
-//
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
-//
-//        EntityIndex ei = eif.createEntityIndex( applicationScope );
-//        ei.initializeIndex();
-//
-//        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
-//
-//        Map entityMap = new HashMap() {{
-//            put( "username", "edanuff" );
-//            put( "email", "ed@anuff.com" );
-//            put( "middlename", middleName );
-//        }};
-//
-//        Entity user = EntityIndexMapUtils.fromMap( entityMap );
-//        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
-//        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-//
-//
-//        EntityIndexBatch batch = ei.createBatch();
-//
-//        batch.index( appScope, user ).executeAndRefresh();
-//        Query query = new Query();
-//        query.addEqualityFilter( "username", "edanuff" );
-//        CandidateResults r = ei.search( appScope, query );
-//        assertEquals( user.getId(), r.get( 0 ).getId() );
-//
-//        batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
-//
-//
-//        // EntityRef
-//        query = new Query();
-//        query.addEqualityFilter( "username", "edanuff" );
-//        r = ei.search(appScope, query );
-//
-//        assertFalse( r.iterator().hasNext() );
-//    }
-//
-//    @Test
-//    public void multiValuedTypes() {
-//
-//        Id appId = new SimpleId( "entityindextest" );
-//        Id ownerId = new SimpleId( "multivaluedtype" );
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
-//
-//        EntityIndex ei = eif.createEntityIndex( applicationScope );
-//        ei.createBatch();
-//
-//        // Bill has favorites as string, age as string and retirement goal as number
-//        Map billMap = new HashMap() {{
-//            put( "username", "bill" );
-//            put( "email", "bill@example.com" );
-//            put( "age", "thirtysomething");
-//            put( "favorites", "scallops, croquet, wine");
-//            put( "retirementGoal", 100000);
-//        }};
-//        Entity bill = EntityIndexMapUtils.fromMap( billMap );
-//        EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
-//        EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
-//
-//        EntityIndexBatch batch = ei.createBatch();
-//
-//        batch.index( appScope,  bill );
-//
-//        // Fred has age as int, favorites as object and retirement goal as object
-//        Map fredMap = new HashMap() {{
-//            put( "username", "fred" );
-//            put( "email", "fred@example.com" );
-//            put( "age", 41 );
-//            put( "favorites", new HashMap<String, Object>() {{
-//                put("food", "cheezewiz");
-//                put("sport", "nascar");
-//                put("beer", "budwizer");
-//            }});
-//            put( "retirementGoal", new HashMap<String, Object>() {{
-//                put("car", "Firebird");
-//                put("home", "Mobile");
-//            }});
-//        }};
-//        Entity fred = EntityIndexMapUtils.fromMap( fredMap );
-//        EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
-//        EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
-//        batch.index( appScope, fred );
-//
-//        batch.executeAndRefresh();
-//
-//        Query query = new Query();
-//        query.addEqualityFilter( "username", "bill" );
-//        CandidateResults r = ei.search( appScope, query );
-//        assertEquals( bill.getId(), r.get( 0 ).getId() );
-//
-//        query = new Query();
-//        query.addEqualityFilter( "username", "fred" );
-//        r = ei.search( appScope,  query );
-//        assertEquals( fred.getId(), r.get( 0 ).getId() );
-//
-//        query = new Query();
-//        query.addEqualityFilter( "age", 41 );
-//        r = ei.search( appScope,  query );
-//        assertEquals( fred.getId(), r.get( 0 ).getId() );
-//
-//        query = new Query();
-//        query.addEqualityFilter( "age", "thirtysomething" );
-//        r = ei.search(  appScope, query );
-//        assertEquals( bill.getId(), r.get( 0 ).getId() );
-//    }
-//
-//
-//    @Test
-//    public void healthTest() {
-//
-//        Id appId = new SimpleId( "application" );
-//        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-//
-//        EntityIndex ei = eif.createEntityIndex( applicationScope );
-//
-//        assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
-//        assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() );
-//
-//        ei.initializeIndex();
-//        ei.refresh();
-//
-//        assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
-//        assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
-//    }
+    private static final Logger log = LoggerFactory.getLogger( EntityIndexTest.class );
+
+    @ClassRule
+    public static CassandraRule cass = new CassandraRule();
+
+    @Rule
+    public ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
+
+    @Inject
+    @Rule
+    public MigrationManagerRule migrationManagerRule;
+
+    @Inject
+    public EntityIndexFactory eif;
+
+
+
+    @Test
+    public void testIndex() throws IOException {
+
+        final int MAX_ENTITIES = 100;
+
+        Id appId = new SimpleId( "application" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( appId, "things" );
+
+        final String entityType = "thing";
+
+
+        final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
+        ObjectMapper mapper = new ObjectMapper();
+        List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
+
+        int count = 0;
+        StopWatch timer = new StopWatch();
+        timer.start();
+
+        final EntityIndexBatch batch = entityIndex.createBatch();
+
+        for ( Object o : sampleJson ) {
+
+            Map<String, Object> item = ( Map<String, Object> ) o;
+
+            Entity entity = new Entity( entityType );
+            entity = EntityIndexMapUtils.fromMap( entity, item );
+            EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+
+            batch.index( indexScope, entity );
+
+            if(count %1000 == 0){
+                batch.execute();
+            }
+
+
+
+            if ( count++ > MAX_ENTITIES ) {
+                break;
+            }
+        }
+
+        batch.execute();
+        timer.stop();
+        log.info( "Total time to index {} entries {}ms, average {}ms/entry",
+                new Object[] { count, timer.getTime(), timer.getTime() / count } );
+
+        entityIndex.refresh();
+
+
+        testQueries( indexScope, searchTypes,  entityIndex );
+    }
+
+
+    @Test
+    public void testDeindex() {
+
+        Id appId = new SimpleId( "application" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        Map entityMap = new HashMap() {{
+            put( "name", "Ferrari 212 Inter" );
+            put( "introduced", 1952 );
+            put( "topspeed", 215 );
+        }};
+
+
+        Entity entity = EntityIndexMapUtils.fromMap( entityMap );
+        EntityUtils.setId( entity, new SimpleId( "fastcar" ) );
+        EntityUtils.setVersion( entity, UUIDGenerator.newTimeUUID() );
+        entityIndex.createBatch().index(indexScope , entity ).executeAndRefresh();
+
+        CandidateResults candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()),
+                Query.fromQL( "name contains 'Ferrari*'" ) );
+        assertEquals( 1, candidateResults.size() );
+
+        entityIndex.createBatch().deindex( indexScope, entity ).execute();
+
+        entityIndex.refresh();
+
+        candidateResults = entityIndex.search( indexScope, SearchTypes.fromTypes(entity.getId().getType()), Query.fromQL( "name contains 'Ferrari*'" ) );
+        assertEquals( 0, candidateResults.size() );
+    }
+
+
+    private void testQuery(final IndexScope scope, final SearchTypes searchTypes, final EntityIndex entityIndex, final String queryString, final int num ) {
+
+        StopWatch timer = new StopWatch();
+        timer.start();
+        Query query = Query.fromQL( queryString );
+        query.setLimit( 1000 );
+        CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query );
+        timer.stop();
+
+        assertEquals( num, candidateResults.size() );
+        log.debug( "Query time {}ms", timer.getTime() );
+    }
+
+
+    private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex entityIndex ) {
+
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan'", 0 );
+
+        testQuery(scope, searchTypes, entityIndex, "name contains 'Morgan'", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "company > 'GeoLogix'", 64 );
+
+        testQuery(scope, searchTypes, entityIndex, "gender = 'female'", 45 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 39 and age < 41", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age > 40", 0 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age >= 40", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Minerva Harrell' and age <= 40", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan* '", 1 );
+
+        testQuery(scope, searchTypes, entityIndex, "name = 'Morgan*'", 1 );
+
+
+        // test a couple of array sub-property queries
+
+        int totalUsers = 102;
+
+        // nobody has a friend named Jack the Ripper
+        testQuery(scope, searchTypes, entityIndex, "friends.name = 'Jack the Ripper'", 0 );
+
+        // everybody doesn't have a friend named Jack the Ripper
+        testQuery(scope,  searchTypes,entityIndex, "not (friends.name = 'Jack the Ripper')", totalUsers );
+
+        // one person has a friend named Shari Hahn
+        testQuery(scope, searchTypes, entityIndex, "friends.name = 'Wendy Moody'", 1 );
+
+        // everybody but 1 doesn't have a friend named Shari Hahh
+        testQuery(scope, searchTypes, entityIndex, "not (friends.name = 'Shari Hahn')", totalUsers - 1);
+
+    }
+
+
+    /**
+     * Tests that Entity-to-map and Map-to-entity round trip works.
+     */
+    @Test
+    public void testEntityIndexMapUtils() throws IOException {
+
+        InputStream is = this.getClass().getResourceAsStream( "/sample-small.json" );
+        ObjectMapper mapper = new ObjectMapper();
+        List<Object> contacts = mapper.readValue( is, new TypeReference<List<Object>>() {} );
+
+        for ( Object o : contacts ) {
+
+            Map<String, Object> map1 = ( Map<String, Object> ) o;
+
+            // convert map to entity
+            Entity entity1 = EntityIndexMapUtils.fromMap( map1 );
+
+            // convert entity back to map
+            Map map2 = EntityIndexMapUtils.toMap( entity1 );
+
+            // the two maps should be the same
+            Map diff = Maps.difference( map1, map2 ).entriesDiffering();
+            assertEquals( 0, diff.size() );
+        }
+    }
+
+
+    @Test
+    public void getEntityVersions() throws Exception {
+
+        Id appId = new SimpleId( "application" );
+        Id ownerId = new SimpleId( "owner" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope indexScope = new IndexScopeImpl( ownerId, "users" );
+
+
+
+        EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
+        entityIndex.initializeIndex();
+
+        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+        Map<String, Object> properties = new LinkedHashMap<String, Object>();
+        properties.put( "username", "edanuff" );
+        properties.put( "email", "ed@anuff.com" );
+        properties.put( "middlename", middleName );
+
+        Map entityMap = new HashMap() {{
+            put( "username", "edanuff" );
+            put( "email", "ed@anuff.com" );
+            put( "middlename", middleName );
+        }};
+
+        final Id userId = new SimpleId("user");
+
+        Entity user = EntityIndexMapUtils.fromMap( entityMap );
+        EntityUtils.setId( user, userId);
+        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+
+
+        final EntityIndexBatch batch = entityIndex.createBatch();
+
+        batch.index( indexScope, user );
+
+        user.setField( new StringField( "address1", "1782 address st" ) );
+        batch.index( indexScope, user );
+        user.setField( new StringField( "address2", "apt 508" ) );
+        batch.index( indexScope,  user );
+        user.setField( new StringField( "address3", "apt 508" ) );
+        batch.index( indexScope,  user );
+        batch.executeAndRefresh();
+
+        CandidateResults results = entityIndex.getEntityVersions(indexScope,  user.getId() );
+
+        assertEquals(1,  results.size());
+        assertEquals( results.get( 0 ).getId(), user.getId() );
+        assertEquals( results.get(0).getVersion(), user.getVersion());
+    }
+
+
+    @Test
+    public void deleteVerification() throws Throwable {
+
+        Id appId = new SimpleId( "application" );
+        Id ownerId = new SimpleId( "owner" );
+
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
+
+        EntityIndex ei = eif.createEntityIndex( applicationScope );
+        ei.initializeIndex();
+
+        final String middleName = "middleName" + UUIDUtils.newTimeUUID();
+
+        Map entityMap = new HashMap() {{
+            put( "username", "edanuff" );
+            put( "email", "ed@anuff.com" );
+            put( "middlename", middleName );
+        }};
+
+        Entity user = EntityIndexMapUtils.fromMap( entityMap );
+        EntityUtils.setId( user, new SimpleId( "edanuff" ) );
+        EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
+
+
+        EntityIndexBatch batch = ei.createBatch();
+
+        batch.index( appScope, user ).executeAndRefresh();
+        Query query = new Query();
+        query.addEqualityFilter( "username", "edanuff" );
+        CandidateResults r = ei.search( appScope, SearchTypes.fromTypes( "edanuff" ), query );
+        assertEquals( user.getId(), r.get( 0 ).getId() );
+
+        batch.deindex(appScope, user.getId(), user.getVersion() ).executeAndRefresh();
+
+
+        // EntityRef
+        query = new Query();
+        query.addEqualityFilter( "username", "edanuff" );
+        r = ei.search(appScope,SearchTypes.fromTypes( "edanuff" ),  query );
+
+        assertFalse( r.iterator().hasNext() );
+    }
+
+    @Test
+    public void multiValuedTypes() {
+
+        Id appId = new SimpleId( "entityindextest" );
+        Id ownerId = new SimpleId( "multivaluedtype" );
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
+
+        EntityIndex ei = eif.createEntityIndex( applicationScope );
+        ei.createBatch();
+
+        // Bill has favorites as string, age as string and retirement goal as number
+        Map billMap = new HashMap() {{
+            put( "username", "bill" );
+            put( "email", "bill@example.com" );
+            put( "age", "thirtysomething");
+            put( "favorites", "scallops, croquet, wine");
+            put( "retirementGoal", 100000);
+        }};
+        Entity bill = EntityIndexMapUtils.fromMap( billMap );
+        EntityUtils.setId( bill, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
+        EntityUtils.setVersion( bill, UUIDGenerator.newTimeUUID() );
+
+        EntityIndexBatch batch = ei.createBatch();
+
+        batch.index( appScope,  bill );
+
+        // Fred has age as int, favorites as object and retirement goal as object
+        Map fredMap = new HashMap() {{
+            put( "username", "fred" );
+            put( "email", "fred@example.com" );
+            put( "age", 41 );
+            put( "favorites", new HashMap<String, Object>() {{
+                put("food", "cheezewiz");
+                put("sport", "nascar");
+                put("beer", "budwizer");
+            }});
+            put( "retirementGoal", new HashMap<String, Object>() {{
+                put("car", "Firebird");
+                put("home", "Mobile");
+            }});
+        }};
+        Entity fred = EntityIndexMapUtils.fromMap( fredMap );
+        EntityUtils.setId( fred, new SimpleId( UUIDGenerator.newTimeUUID(), "user"  ) );
+        EntityUtils.setVersion( fred, UUIDGenerator.newTimeUUID() );
+        batch.index( appScope, fred );
+
+        batch.executeAndRefresh();
+
+        final SearchTypes searchTypes = SearchTypes.fromTypes( "user" );
+
+        Query query = new Query();
+        query.addEqualityFilter( "username", "bill" );
+        CandidateResults r = ei.search( appScope, searchTypes,  query );
+        assertEquals( bill.getId(), r.get( 0 ).getId() );
+
+        query = new Query();
+        query.addEqualityFilter( "username", "fred" );
+        r = ei.search( appScope, searchTypes,  query );
+        assertEquals( fred.getId(), r.get( 0 ).getId() );
+
+        query = new Query();
+        query.addEqualityFilter( "age", 41 );
+        r = ei.search( appScope, searchTypes,  query );
+        assertEquals( fred.getId(), r.get( 0 ).getId() );
+
+        query = new Query();
+        query.addEqualityFilter( "age", "thirtysomething" );
+        r = ei.search(  appScope, searchTypes, query );
+        assertEquals( bill.getId(), r.get( 0 ).getId() );
+    }
+
+
+    @Test
+    public void healthTest() {
+
+        Id appId = new SimpleId( "application" );
+        ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
+
+        EntityIndex ei = eif.createEntityIndex( applicationScope );
+
+        assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
+        assertEquals( "index not be ready yet", Health.RED, ei.getIndexHealth() );
+
+        ei.initializeIndex();
+        ei.refresh();
+
+        assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
+        assertNotEquals( "cluster should be ready now", Health.RED, ei.getClusterHealth() );
+    }
 }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
index c972851..2521a38 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
@@ -36,7 +36,7 @@ public class EsRunner extends ITRunner {
         ElasticSearchResource rule = new ElasticSearchResource();
 
         try {
-            rule.before();
+//            rule.before();
         }
         catch ( Throwable throwable ) {
             throw new RuntimeException( "Unable to start ES", throwable );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/ef77a5a4/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index d0e2d11..34185b3 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -429,7 +429,7 @@ public class GrammarTreeTest {
         assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
         assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
 
-        QueryBuilder qb = query.createQueryBuilder();
+        QueryBuilder qb = query.createQueryBuilder("testcontext");
     }
 
 


[16/27] incubator-usergrid git commit: Merge branch 'es-type-updates' into es-type-updates-ryan-external

Posted by to...@apache.org.
Merge branch 'es-type-updates' into es-type-updates-ryan-external

Conflicts:
	stack/loadtests/README.md


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/e16dd43d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/e16dd43d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/e16dd43d

Branch: refs/heads/two-dot-o
Commit: e16dd43db77c44c27736b311c07b80f04ea62ff9
Parents: be6c3c2 c31e1b3
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 16:28:41 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 16:28:41 2014 -0700

----------------------------------------------------------------------
 stack/awscluster/src/main/cql/create_locks.cql  |  26 --
 .../awscluster/src/main/cql/create_usergrid.cql |  28 ---
 .../main/cql/create_usergrid_applications.cql   |  47 ----
 stack/awscluster/src/main/cql/update_locks.cql  |  23 ++
 .../awscluster/src/main/cql/update_usergrid.cql |  27 ++
 .../main/cql/update_usergrid_applications.cql   |  46 ++++
 .../main/dist/init_instance/create_keyspaces.sh |  61 -----
 .../main/dist/init_instance/init_db_server.sh   |   4 -
 .../main/dist/init_instance/init_rest_server.sh |  28 ++-
 .../main/dist/init_instance/install_yourkit.sh  |   6 +-
 .../main/dist/init_instance/update_keyspaces.sh |  66 +++++
 .../src/main/groovy/NodeRegistry.groovy         |   8 +-
 .../src/main/groovy/configure_usergrid.groovy   |   2 +-
 stack/awscluster/ugcluster-cf.json              | 246 ++++++++++++++-----
 stack/core/pom.xml                              |   1 +
 .../corepersistence/CpEntityManager.java        |   5 +-
 .../corepersistence/CpRelationManager.java      |  25 +-
 .../usergrid/corepersistence/CpSetup.java       |   4 +-
 .../persistence/cassandra/CassandraService.java |   3 +
 stack/loadtests/README.md                       |  16 +-
 .../datagenerators/EntityDataGenerator.scala    |  23 +-
 .../datagenerators/FeederGenerator.scala        |  52 +++-
 .../org/apache/usergrid/helpers/Setup.scala     |  46 ++--
 .../scenarios/ConnectionScenarios.scala         |  29 ++-
 .../usergrid/scenarios/EntityScenarios.scala    |  65 +++++
 .../scenarios/NotificationScenarios.scala       |  15 +-
 .../usergrid/scenarios/UserScenarios.scala      |  32 ++-
 .../org/apache/usergrid/settings/Settings.scala |  34 +--
 .../usergrid/simulations/AppSimulation.scala    |   9 +-
 .../simulations/ConnectionsSimulation.scala     |  53 ++++
 .../simulations/GeoProfileSimulation.scala      |  33 ---
 .../PostCustomEntitySimulation.scala            |  61 +++++
 .../simulations/PostDevicesSimulation.scala     |  45 ----
 .../usergrid/simulations/SetupSimulation.scala  |  45 ++++
 stack/loadtests/src/test/scala/Engine.scala     |   4 +-
 .../services/users/devices/DevicesService.java  |   4 +-
 36 files changed, 818 insertions(+), 404 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/e16dd43d/stack/loadtests/README.md
----------------------------------------------------------------------


[27/27] incubator-usergrid git commit: Merge branch 'es-type-updates-ryan-external' into two-dot-o

Posted by to...@apache.org.
Merge branch 'es-type-updates-ryan-external' into two-dot-o


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/d2d1fe74
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/d2d1fe74
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/d2d1fe74

Branch: refs/heads/two-dot-o
Commit: d2d1fe74fef4c02c14d8149355620957aa453ab7
Parents: 84b8365 28eb2cd
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 12 12:11:31 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 12 12:11:31 2014 -0700

----------------------------------------------------------------------
 stack/awscluster/src/main/cql/create_locks.cql  |  26 -
 .../awscluster/src/main/cql/create_usergrid.cql |  28 -
 .../main/cql/create_usergrid_applications.cql   |  47 --
 stack/awscluster/src/main/cql/update_locks.cql  |  23 +
 .../awscluster/src/main/cql/update_usergrid.cql |  27 +
 .../main/cql/update_usergrid_applications.cql   |  46 ++
 .../main/dist/init_instance/create_keyspaces.sh |  61 --
 .../main/dist/init_instance/init_db_server.sh   |   4 -
 .../main/dist/init_instance/init_rest_server.sh |  28 +-
 .../main/dist/init_instance/install_yourkit.sh  |   6 +-
 .../main/dist/init_instance/update_keyspaces.sh |  66 ++
 .../src/main/groovy/NodeRegistry.groovy         |   8 +-
 .../src/main/groovy/configure_usergrid.groovy   |   2 +-
 stack/awscluster/ugcluster-cf.json              | 246 +++++--
 .../main/resources/usergrid-default.properties  |   2 +-
 .../src/test/resources/usergrid-test.properties |   6 +-
 .../CpEntityIndexDeleteListener.java            |   2 +-
 .../corepersistence/CpEntityManager.java        |  40 +-
 .../corepersistence/CpRelationManager.java      | 533 +++++++--------
 .../usergrid/corepersistence/CpSetup.java       |   4 +-
 .../results/FilteringLoader.java                |  26 +-
 .../results/ResultsLoaderFactory.java           |   9 +-
 .../results/ResultsLoaderFactoryImpl.java       |   6 +-
 .../corepersistence/util/CpNamingUtils.java     |  16 +-
 .../persistence/cassandra/CassandraService.java |   3 +
 .../cassandra/RelationManagerImpl.java          |  18 +-
 .../apache/usergrid/ConcurrentCoreITSuite.java  |   2 +-
 .../usergrid/ConcurrentCoreIteratorITSuite.java |   4 +-
 .../org/apache/usergrid/CoreApplication.java    |   3 +-
 .../org/apache/usergrid/CoreITSetupImpl.java    |   1 -
 .../java/org/apache/usergrid/CoreITSuite.java   |   4 +-
 .../batch/job/AbstractSchedulerRuntimeIT.java   |   7 +-
 .../usergrid/batch/job/SchedulerRuntime1IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime2IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime3IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime4IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime5IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime6IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime7IT.java |   2 +
 .../usergrid/batch/job/SchedulerRuntime8IT.java |   2 +-
 .../corepersistence/StaleIndexCleanupTest.java  | 166 ++---
 .../org/apache/usergrid/persistence/GeoIT.java  |  34 +-
 .../apache/usergrid/persistence/QueryTest.java  |  15 +
 .../resources/usergrid-custom-test.properties   |   4 +-
 .../data/MigrationInfoSerializationImpl.java    |   2 +-
 .../core/astyanax/ColumnNameIteratorTest.java   |  21 +-
 .../MultiKeyColumnNameIteratorTest.java         |  20 +-
 .../astyanax/MultiRowColumnIteratorTest.java    |  20 +-
 .../core/cassandra/CassandraRule.java           | 114 ----
 .../data/MigrationInfoSerializationTest.java    |  13 +-
 .../persistence/core/test/ITRunner.java         |  20 +-
 .../graph/CommittedGraphManagerIT.java          |   2 +-
 .../persistence/graph/GraphManagerLoadTest.java |   4 +-
 .../graph/GraphManagerShardConsistencyIT.java   |   6 -
 .../graph/GraphManagerShardingIT.java           |   4 +-
 .../graph/GraphManagerStressTest.java           |   4 +-
 .../graph/StorageGraphManagerIT.java            |   2 +-
 .../graph/guice/TestGraphModule.java            |   2 +-
 .../graph/impl/EdgeDeleteListenerTest.java      |   6 +-
 .../graph/impl/NodeDeleteListenerTest.java      |   6 +-
 .../graph/impl/stage/EdgeDeleteRepairTest.java  |   4 +-
 .../graph/impl/stage/EdgeMetaRepairTest.java    |   6 +-
 .../EdgeMetaDataSerializationProxyV2Test.java   |   7 +-
 .../EdgeMetaDataSerializationV1Test.java        |   4 +-
 .../EdgeMetaDataSerializationV2Test.java        |   4 +-
 .../EdgeSerializationChopTest.java              |   4 +-
 .../serialization/EdgeSerializationTest.java    |   7 +-
 .../serialization/NodeSerializationTest.java    |   4 +-
 .../PermanentSerializationTest.java             |   2 +-
 .../impl/shard/EdgeShardSerializationTest.java  |   9 +-
 .../NodeShardCounterSerializationTest.java      |   8 +-
 stack/corepersistence/pom.xml                   |   2 +-
 .../usergrid/persistence/index/EntityIndex.java |   4 +-
 .../usergrid/persistence/index/SearchTypes.java | 130 ++++
 .../index/impl/EsEntityIndexBatchImpl.java      | 112 ++--
 .../index/impl/EsEntityIndexImpl.java           | 179 +++--
 .../persistence/index/impl/EsQueryVistor.java   |   3 +-
 .../persistence/index/impl/IndexScopeImpl.java  |  14 +-
 .../persistence/index/impl/IndexingUtils.java   | 107 ++-
 .../usergrid/persistence/index/query/Query.java |  76 ++-
 .../index/impl/CorePerformanceIT.java           |  40 +-
 .../index/impl/ElasticSearchResource.java       | 115 +---
 .../impl/EntityConnectionIndexImplTest.java     | 239 ++++++-
 .../persistence/index/impl/EntityIndexTest.java | 115 ++--
 .../persistence/query/tree/GrammarTreeTest.java |   2 +-
 .../src/test/resources/dynamic-test.properties  |   2 +-
 .../src/test/resources/usergrid-UNIT.properties |   2 +-
 .../usergrid/launcher/EmbeddedServerHelper.java |   4 +-
 stack/loadtests/README.md                       |   2 +-
 stack/pom.xml                                   |   7 -
 .../apache/usergrid/rest/AbstractRestIT.java    |  22 +-
 .../java/org/apache/usergrid/rest/ITSetup.java  |  16 +-
 .../org/apache/usergrid/rest/RestITSuite.java   |   2 +-
 .../apache/usergrid/rest/TestContextSetup.java  |  10 +-
 .../rest/management/OrganizationsIT.java        | 328 +++++-----
 .../rest/management/RegistrationIT.java         |  61 +-
 .../usergrid/rest/test/security/TestUser.java   |   3 +
 .../resources/usergrid-custom-test.properties   |   4 +-
 .../org/apache/usergrid/ServiceITSuite.java     |   2 +-
 .../resources/usergrid-custom-test.properties   |   4 +-
 stack/test-utils/pom.xml                        |   5 -
 .../usergrid/cassandra/CassandraResource.java   | 648 ++++---------------
 .../cassandra/CassandraResourceTest.java        |   7 +-
 .../resources/usergrid-custom-test.properties   |   2 +-
 104 files changed, 2041 insertions(+), 2072 deletions(-)
----------------------------------------------------------------------



[04/27] incubator-usergrid git commit: Changed implementation to remove all types and instead use SearchTypes

Posted by to...@apache.org.
Changed implementation to remove all types and instead use SearchTypes


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/eef2b70b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/eef2b70b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/eef2b70b

Branch: refs/heads/two-dot-o
Commit: eef2b70bfd1c2e97aa446ae683645b36215e2a1a
Parents: 1debf19
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 7 08:54:47 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 7 08:54:47 2014 -0700

----------------------------------------------------------------------
 .../CpEntityIndexDeleteListener.java            |   2 +-
 .../corepersistence/CpEntityManager.java        |  46 +-
 .../corepersistence/CpRelationManager.java      | 482 +++++++++----------
 .../results/FilteringLoader.java                |   2 +-
 .../corepersistence/util/CpNamingUtils.java     |  14 +-
 .../corepersistence/StaleIndexCleanupTest.java  |  17 +-
 .../usergrid/persistence/index/SearchTypes.java |  52 ++
 .../persistence/index/impl/EsRunner.java        |   2 +-
 8 files changed, 325 insertions(+), 292 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
index 13becc4..1999702 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
@@ -65,7 +65,7 @@ public class CpEntityIndexDeleteListener {
 
         final CollectionScope collectionScope = event.getCollectionScope();
         final IndexScope indexScope = 
-                new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName(), entityType );
+                new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName() );
         final EntityIndex entityIndex = entityIndexFactory.createEntityIndex(
                 new ApplicationScopeImpl( collectionScope.getApplication()));
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 97bdba8..00a83a4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -637,7 +637,7 @@ public class CpEntityManager implements EntityManager {
 
                         IndexScope indexScope = new IndexScopeImpl(
                                 new SimpleId( uuid, ownerType ), 
-                                CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ), entityType );
+                                CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ) );
 
 
                         batch.index( indexScope, entity );
@@ -650,15 +650,16 @@ public class CpEntityManager implements EntityManager {
             // deindex from default index scope
             IndexScope defaultIndexScope = new IndexScopeImpl(
                     getApplicationScope().getApplication(),
-                    CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ), entityType );
+                    CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
 
             batch.deindex(defaultIndexScope,  entity );
 
-            IndexScope allTypesIndexScope = new IndexScopeImpl(
-                getApplicationScope().getApplication(), 
-                    CpNamingUtils.ALL_TYPES, entityType );
-
-            batch.deindex( allTypesIndexScope,  entity );
+//            TODO REMOVE INDEX CODE
+//            IndexScope allTypesIndexScope = new IndexScopeImpl(
+//                getApplicationScope().getApplication(),
+//                    CpNamingUtils.ALL_TYPES, entityType );
+//
+//            batch.deindex( allTypesIndexScope,  entity );
 
             batch.execute();
 
@@ -1048,7 +1049,7 @@ public class CpEntityManager implements EntityManager {
 
         IndexScope defaultIndexScope = new IndexScopeImpl(
                 getApplicationScope().getApplication(), 
-                CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ), entityType );
+                CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
 
         EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
         EntityIndex ei = managerCache.getEntityIndex(getApplicationScope());
@@ -2976,23 +2977,24 @@ public class CpEntityManager implements EntityManager {
         // index member into entity collection | type scope
         IndexScope collectionIndexScope = new IndexScopeImpl(
                 collectionEntity.getId(),
-                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
+                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
 
         batch.index(collectionIndexScope, memberEntity);
-        
-        // index member into entity | all-types scope
-        IndexScope entityAllTypesScope = new IndexScopeImpl(
-                collectionEntity.getId(),
-                CpNamingUtils.ALL_TYPES, entityType );
 
-        batch.index(entityAllTypesScope, memberEntity);
-        
-        // index member into application | all-types scope
-        IndexScope appAllTypesScope = new IndexScopeImpl(
-                getApplicationScope().getApplication(),
-                CpNamingUtils.ALL_TYPES, entityType );
-
-        batch.index(appAllTypesScope, memberEntity);
+        //TODO REMOVE INDEX CODE
+//        // index member into entity | all-types scope
+//        IndexScope entityAllTypesScope = new IndexScopeImpl(
+//                collectionEntity.getId(),
+//                CpNamingUtils.ALL_TYPES, entityType );
+//
+//        batch.index(entityAllTypesScope, memberEntity);
+//
+//        // index member into application | all-types scope
+//        IndexScope appAllTypesScope = new IndexScopeImpl(
+//                getApplicationScope().getApplication(),
+//                CpNamingUtils.ALL_TYPES, entityType );
+//
+//        batch.index(appAllTypesScope, memberEntity);
 
         batch.execute();
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c33c032..c822679 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -78,6 +78,7 @@ import org.apache.usergrid.persistence.graph.impl.SimpleSearchEdgeType;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
@@ -190,11 +191,11 @@ public class CpRelationManager implements RelationManager {
     public CpRelationManager() {}
 
 
-    public CpRelationManager init( 
-        EntityManager em, 
+    public CpRelationManager init(
+        EntityManager em,
             CpEntityManagerFactory emf,
             UUID applicationId,
-            EntityRef headEntity, 
+            EntityRef headEntity,
             IndexBucketLocator indexBucketLocator ) {
 
         Assert.notNull( em, "Entity manager cannot be null" );
@@ -217,8 +218,8 @@ public class CpRelationManager implements RelationManager {
         this.indexBucketLocator = indexBucketLocator; // TODO: this also
 
         // load the Core Persistence version of the head entity as well
-        this.headEntityScope = new CollectionScopeImpl( 
-            this.applicationScope.getApplication(), 
+        this.headEntityScope = new CollectionScopeImpl(
+            this.applicationScope.getApplication(),
             this.applicationScope.getApplication(),
             CpNamingUtils.getCollectionScopeNameFromEntityType( headEntity.getType() ) );
 
@@ -226,7 +227,7 @@ public class CpRelationManager implements RelationManager {
         if ( logger.isDebugEnabled() ) {
             logger.debug( "Loading head entity {}:{} from scope\n   app {}\n   owner {}\n   name {}",
                 new Object[] {
-                    headEntity.getType(), 
+                    headEntity.getType(),
                     headEntity.getUuid(),
                     headEntityScope.getApplication(),
                     headEntityScope.getOwner(),
@@ -235,7 +236,7 @@ public class CpRelationManager implements RelationManager {
         }
 
         Id entityId = new SimpleId( headEntity.getUuid(), headEntity.getType() );
-        this.cpHeadEntity = ((CpEntityManager)em).load( 
+        this.cpHeadEntity = ((CpEntityManager)em).load(
             new CpEntityManager.EntityScope( headEntityScope, entityId));
 
         // commented out because it is possible that CP entity has not been created yet
@@ -255,12 +256,12 @@ public class CpRelationManager implements RelationManager {
 
         String edgeTypePrefix = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
 
-        logger.debug("getCollectionIndexes(): Searching for edge type prefix {} to target {}:{}", 
+        logger.debug("getCollectionIndexes(): Searching for edge type prefix {} to target {}:{}",
             new Object[] {
                 edgeTypePrefix, cpHeadEntity.getId().getType(), cpHeadEntity.getId().getUuid()
         });
 
-        Observable<String> types= gm.getEdgeTypesFromSource( 
+        Observable<String> types= gm.getEdgeTypesFromSource(
             new SimpleSearchEdgeType( cpHeadEntity.getId(), edgeTypePrefix,  null ));
 
         Iterator<String> iter = types.toBlockingObservable().getIterator();
@@ -278,7 +279,7 @@ public class CpRelationManager implements RelationManager {
         //Map<EntityRef, Set<String>> containerEntities = getContainers(-1, "owns", null);
         Map<EntityRef, Set<String>> containerEntities = getContainers();
 
-        Map<String, Map<UUID, Set<String>>> owners = 
+        Map<String, Map<UUID, Set<String>>> owners =
                 new LinkedHashMap<String, Map<UUID, Set<String>>>();
 
         for ( EntityRef owner : containerEntities.keySet() ) {
@@ -309,7 +310,7 @@ public class CpRelationManager implements RelationManager {
 
         GraphManager gm = managerCache.getGraphManager(applicationScope);
 
-        Iterator<String> edgeTypes = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( 
+        Iterator<String> edgeTypes = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType(
             cpHeadEntity.getId(), edgeType, null) ).toBlocking().getIterator();
 
         logger.debug("getContainers(): "
@@ -319,14 +320,14 @@ public class CpRelationManager implements RelationManager {
                 cpHeadEntity.getId().getType(),
                 cpHeadEntity.getId().getUuid(),
                 applicationScope.getApplication(),
-                edgeTypes.hasNext() 
+                edgeTypes.hasNext()
         });
 
         while ( edgeTypes.hasNext() ) {
 
             String etype = edgeTypes.next();
 
-            Observable<Edge> edges = gm.loadEdgesToTarget( new SimpleSearchByEdgeType( 
+            Observable<Edge> edges = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
                 cpHeadEntity.getId(), etype, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ));
 
             Iterator<Edge> iter = edges.toBlockingObservable().getIterator();
@@ -338,7 +339,7 @@ public class CpRelationManager implements RelationManager {
                     continue;
                 }
 
-                EntityRef eref = new SimpleEntityRef( 
+                EntityRef eref = new SimpleEntityRef(
                         edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
 
                 String name = null;
@@ -365,13 +366,13 @@ public class CpRelationManager implements RelationManager {
 
         final GraphManager gm = managerCache.getGraphManager( applicationScope );
 
-        Iterator<String> edgeTypesToTarget = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType( 
+        Iterator<String> edgeTypesToTarget = gm.getEdgeTypesToTarget( new SimpleSearchEdgeType(
             cpHeadEntity.getId(), null, null) ).toBlockingObservable().getIterator();
 
         logger.debug("updateContainingCollectionsAndCollections(): "
-                + "Searched for edges to target {}:{}\n   in scope {}\n   found: {}", 
+                + "Searched for edges to target {}:{}\n   in scope {}\n   found: {}",
             new Object[] {
-                cpHeadEntity.getId().getType(), 
+                cpHeadEntity.getId().getType(),
                 cpHeadEntity.getId().getUuid(),
                 applicationScope.getApplication(),
                 edgeTypesToTarget.hasNext()
@@ -391,8 +392,8 @@ public class CpRelationManager implements RelationManager {
                 .flatMap( new Func1<String, Observable<Edge>>() {
                     @Override
                     public Observable<Edge> call( final String etype ) {
-                        return gm.loadEdgesToTarget( new SimpleSearchByEdgeType( 
-                            cpHeadEntity.getId(), etype, Long.MAX_VALUE, 
+                        return gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
+                            cpHeadEntity.getId(), etype, Long.MAX_VALUE,
                             SearchByEdgeType.Order.DESCENDING, null ) );
                     }
                 } )
@@ -402,8 +403,8 @@ public class CpRelationManager implements RelationManager {
                     @Override
                     public void call( final Edge edge ) {
 
-                        EntityRef sourceEntity = new SimpleEntityRef( 
-                                edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
+                        EntityRef sourceEntity =
+                                new SimpleEntityRef( edge.getSourceNode().getType(), edge.getSourceNode().getUuid() );
 
                         // reindex the entity in the source entity's collection or connection index
 
@@ -411,29 +412,30 @@ public class CpRelationManager implements RelationManager {
                         if ( CpNamingUtils.isCollectionEdgeType( edge.getType() ) ) {
 
                             String collName = CpNamingUtils.getCollectionName( edge.getType() );
-                            indexScope = new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(),
-                                sourceEntity.getType() ),
-                                CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
+                            indexScope =
+                                    new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(), sourceEntity.getType() ),
+                                            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
                         }
                         else {
 
                             String connName = CpNamingUtils.getCollectionName( edge.getType() );
-                            indexScope = new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(),
-                                sourceEntity.getType() ),
-                                CpNamingUtils.getConnectionScopeName( cpEntity.getId().getType(),
-                                connName ), entityType );
+                            indexScope =
+                                    new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(), sourceEntity.getType() ),
+                                            CpNamingUtils.getConnectionScopeName( connName ) );
                         }
 
                         entityIndexBatch.index( indexScope, cpEntity );
 
                         // reindex the entity in the source entity's all-types index
 
-                        indexScope = new IndexScopeImpl( new SimpleId( 
-                            sourceEntity.getUuid(), sourceEntity.getType() ), CpNamingUtils.ALL_TYPES, entityType );
-
-                        entityIndexBatch.index( indexScope, cpEntity );
+                        //TODO REMOVE INDEX CODE
+                        //                        indexScope = new IndexScopeImpl( new SimpleId(
+                        //                            sourceEntity.getUuid(), sourceEntity.getType() ), CpNamingUtils
+                        // .ALL_TYPES, entityType );
+                        //
+                        //                        entityIndexBatch.index( indexScope, cpEntity );
                     }
-            } ).count().toBlocking().lastOrDefault( 0 );
+                } ).count().toBlocking().lastOrDefault( 0 );
 
 
         entityIndexBatch.execute();
@@ -449,19 +451,19 @@ public class CpRelationManager implements RelationManager {
 
         String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType );
 
-        logger.debug("isConnectionMember(): Checking for edge type {} from {}:{} to {}:{}", 
-            new Object[] { 
+        logger.debug("isConnectionMember(): Checking for edge type {} from {}:{} to {}:{}",
+            new Object[] {
                 edgeType,
-                headEntity.getType(), headEntity.getUuid(), 
+                headEntity.getType(), headEntity.getUuid(),
                 entity.getType(), entity.getUuid() });
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
-        Observable<Edge> edges = gm.loadEdgeVersions( new SimpleSearchByEdge( 
-            new SimpleId( headEntity.getUuid(), headEntity.getType() ), 
+        Observable<Edge> edges = gm.loadEdgeVersions( new SimpleSearchByEdge(
+            new SimpleId( headEntity.getUuid(), headEntity.getType() ),
             edgeType,
             entityId,
             Long.MAX_VALUE,
-            SearchByEdgeType.Order.DESCENDING, 
+            SearchByEdgeType.Order.DESCENDING,
             null ) );
 
         return edges.toBlockingObservable().firstOrDefault( null ) != null;
@@ -477,15 +479,15 @@ public class CpRelationManager implements RelationManager {
 
         String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collName );
 
-        logger.debug("isCollectionMember(): Checking for edge type {} from {}:{} to {}:{}", 
-            new Object[] { 
+        logger.debug("isCollectionMember(): Checking for edge type {} from {}:{} to {}:{}",
+            new Object[] {
                 edgeType,
-                headEntity.getType(), headEntity.getUuid(), 
+                headEntity.getType(), headEntity.getUuid(),
                 entity.getType(), entity.getUuid() });
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
-        Observable<Edge> edges = gm.loadEdgeVersions( new SimpleSearchByEdge( 
-            new SimpleId( headEntity.getUuid(), headEntity.getType() ), 
+        Observable<Edge> edges = gm.loadEdgeVersions( new SimpleSearchByEdge(
+            new SimpleId( headEntity.getUuid(), headEntity.getType() ),
             edgeType,
             entityId,
             Long.MAX_VALUE,
@@ -502,11 +504,11 @@ public class CpRelationManager implements RelationManager {
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
 
-        Observable<Edge> edgesToTarget = gm.loadEdgesToTarget( new SimpleSearchByEdgeType( 
+        Observable<Edge> edgesToTarget = gm.loadEdgesToTarget( new SimpleSearchByEdgeType(
             targetId,
             CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
             System.currentTimeMillis(),
-            SearchByEdgeType.Order.DESCENDING, 
+            SearchByEdgeType.Order.DESCENDING,
             null ) ); // last
 
         Iterator<Edge> iterator = edgesToTarget.toBlockingObservable().getIterator();
@@ -527,7 +529,7 @@ public class CpRelationManager implements RelationManager {
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
 
-        Observable<Edge> edgesFromSource = gm.loadEdgesFromSource( new SimpleSearchByEdgeType( 
+        Observable<Edge> edgesFromSource = gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
             sourceId,
             CpNamingUtils.getEdgeTypeFromConnectionType( connectionType ),
             System.currentTimeMillis(),
@@ -547,7 +549,7 @@ public class CpRelationManager implements RelationManager {
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
 
-        Observable<String> str = gm.getEdgeTypesFromSource( 
+        Observable<String> str = gm.getEdgeTypesFromSource(
                 new SimpleSearchEdgeType( cpHeadEntity.getId(), null, null ) );
 
         Iterator<String> iter = str.toBlockingObservable().getIterator();
@@ -590,13 +592,13 @@ public class CpRelationManager implements RelationManager {
     @Override
     public Entity addToCollection( String collName, EntityRef itemRef ) throws Exception {
 
-        CollectionInfo collection = 
+        CollectionInfo collection =
                 getDefaultSchema().getCollection( headEntity.getType(), collName );
         if ( ( collection != null ) && !collection.getType().equals( itemRef.getType() ) ) {
             return null;
         }
 
-        return addToCollection( collName, itemRef, 
+        return addToCollection( collName, itemRef,
                 ( collection != null && collection.getLinkedCollection() != null ) );
     }
 
@@ -622,15 +624,15 @@ public class CpRelationManager implements RelationManager {
         }
 
         // load the new member entity to be added to the collection from its default scope
-        CollectionScope memberScope = new CollectionScopeImpl( 
+        CollectionScope memberScope = new CollectionScopeImpl(
                 applicationScope.getApplication(),
                 applicationScope.getApplication(),
                 CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
         EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager( memberScope );
 
         //TODO, this double load should disappear once events are in
-        Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() ); 
-        org.apache.usergrid.persistence.model.entity.Entity memberEntity = 
+        Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
+        org.apache.usergrid.persistence.model.entity.Entity memberEntity =
             ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId));
 
         if ( memberEntity == null ) {
@@ -639,7 +641,7 @@ public class CpRelationManager implements RelationManager {
         }
 
         if ( logger.isDebugEnabled() ) {
-            logger.debug( "Loaded member entity {}:{} from scope\n   app {}\n   " 
+            logger.debug( "Loaded member entity {}:{} from scope\n   app {}\n   "
                 + "owner {}\n   name {} data {}",
                 new Object[] {
                     itemRef.getType(),
@@ -653,8 +655,8 @@ public class CpRelationManager implements RelationManager {
 
         String edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collName );
 
-        UUID timeStampUuid = memberEntity.getId().getUuid() != null 
-                && UUIDUtils.isTimeBased( memberEntity.getId().getUuid() ) 
+        UUID timeStampUuid = memberEntity.getId().getUuid() != null
+                && UUIDUtils.isTimeBased( memberEntity.getId().getUuid() )
                 ?  memberEntity.getId().getUuid() : UUIDUtils.newTimeUUID();
 
         long uuidHash = UUIDUtils.getUUIDLong( timeStampUuid );
@@ -664,8 +666,8 @@ public class CpRelationManager implements RelationManager {
         GraphManager gm = managerCache.getGraphManager( applicationScope );
         gm.writeEdge( edge ).toBlockingObservable().last();
 
-        logger.debug( "Wrote edgeType {}\n   from {}:{}\n   to {}:{}\n   scope {}:{}", 
-            new Object[] { 
+        logger.debug( "Wrote edgeType {}\n   from {}:{}\n   to {}:{}\n   scope {}:{}",
+            new Object[] {
                 edgeType,
                 cpHeadEntity.getId().getType(),
                 cpHeadEntity.getId().getUuid(),
@@ -687,7 +689,7 @@ public class CpRelationManager implements RelationManager {
         //            headEntityScope.getName()});
 
         if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
-            getRelationManager( itemEntity ).addToCollection( 
+            getRelationManager( itemEntity ).addToCollection(
                     collection.getLinkedCollection(), headEntity, false );
         }
 
@@ -709,7 +711,7 @@ public class CpRelationManager implements RelationManager {
 
     @Override
     @Metered( group = "core", name = "RelationManager_createItemInCollection" )
-    public Entity createItemInCollection( 
+    public Entity createItemInCollection(
         String collName, String itemType, Map<String, Object> properties) throws Exception {
 
         if ( headEntity.getUuid().equals( applicationId ) ) {
@@ -728,7 +730,7 @@ public class CpRelationManager implements RelationManager {
             return em.create( itemType, properties );
         }
 
-        else if ( headEntity.getType().equals( Group.ENTITY_TYPE ) 
+        else if ( headEntity.getType().equals( Group.ENTITY_TYPE )
                 && ( collName.equals( COLLECTION_ROLES ) ) ) {
             UUID groupId = headEntity.getUuid();
             String roleName = ( String ) properties.get( PROPERTY_NAME );
@@ -778,15 +780,15 @@ public class CpRelationManager implements RelationManager {
         }
 
         // load the entity to be removed to the collection
-        CollectionScope memberScope = new CollectionScopeImpl( 
+        CollectionScope memberScope = new CollectionScopeImpl(
                 this.applicationScope.getApplication(),
                 this.applicationScope.getApplication(),
                 CpNamingUtils.getCollectionScopeNameFromEntityType( itemRef.getType() ) );
         EntityCollectionManager memberMgr = managerCache.getEntityCollectionManager( memberScope );
 
         if ( logger.isDebugEnabled() ) {
-            logger.debug( "Loading entity to remove from collection " 
-                + "{}:{} from scope\n   app {}\n   owner {}\n   name {}", 
+            logger.debug( "Loading entity to remove from collection "
+                + "{}:{} from scope\n   app {}\n   owner {}\n   name {}",
                 new Object[] {
                     itemRef.getType(),
                     itemRef.getUuid(),
@@ -796,8 +798,8 @@ public class CpRelationManager implements RelationManager {
                });
         }
 
-        Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() ); 
-        org.apache.usergrid.persistence.model.entity.Entity memberEntity = 
+        Id entityId = new SimpleId( itemRef.getUuid(), itemRef.getType() );
+        org.apache.usergrid.persistence.model.entity.Entity memberEntity =
             ((CpEntityManager)em).load( new CpEntityManager.EntityScope( memberScope, entityId));
 
         final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
@@ -805,16 +807,16 @@ public class CpRelationManager implements RelationManager {
 
         // remove item from collection index
         IndexScope indexScope = new IndexScopeImpl(
-            cpHeadEntity.getId(), 
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
+            cpHeadEntity.getId(),
+            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
 
         batch.deindex( indexScope, memberEntity );
 
         // remove collection from item index 
         IndexScope itemScope = new IndexScopeImpl(
-            memberEntity.getId(), 
+            memberEntity.getId(),
             CpNamingUtils.getCollectionScopeNameFromCollectionName(
-                    Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ), entityType );
+                    Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ) );
 
 
         batch.deindex( itemScope, cpHeadEntity );
@@ -823,18 +825,18 @@ public class CpRelationManager implements RelationManager {
 
         // remove edge from collection to item 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
-        Edge collectionToItemEdge = new SimpleEdge( 
-                cpHeadEntity.getId(), 
-                CpNamingUtils.getEdgeTypeFromCollectionName( collName ), 
+        Edge collectionToItemEdge = new SimpleEdge(
+                cpHeadEntity.getId(),
+                CpNamingUtils.getEdgeTypeFromCollectionName( collName ),
                 memberEntity.getId(), UUIDUtils.getUUIDLong( memberEntity.getId().getUuid() ) );
         gm.deleteEdge( collectionToItemEdge ).toBlockingObservable().last();
 
         // remove edge from item to collection
-        Edge itemToCollectionEdge = new SimpleEdge( 
-                memberEntity.getId(), 
-                CpNamingUtils.getEdgeTypeFromCollectionName( 
-                    Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ), 
-                cpHeadEntity.getId(), 
+        Edge itemToCollectionEdge = new SimpleEdge(
+                memberEntity.getId(),
+                CpNamingUtils.getEdgeTypeFromCollectionName(
+                        Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ),
+                cpHeadEntity.getId(),
                 UUIDUtils.getUUIDLong( cpHeadEntity.getId().getUuid() ) );
 
         gm.deleteEdge( itemToCollectionEdge ).toBlockingObservable().last();
@@ -859,16 +861,16 @@ public class CpRelationManager implements RelationManager {
 
 
     @Override
-    public void copyRelationships(String srcRelationName, EntityRef dstEntityRef, 
+    public void copyRelationships(String srcRelationName, EntityRef dstEntityRef,
             String dstRelationName) throws Exception {
 
         headEntity = em.validate( headEntity );
         dstEntityRef = em.validate( dstEntityRef );
 
-        CollectionInfo srcCollection = 
+        CollectionInfo srcCollection =
                 getDefaultSchema().getCollection( headEntity.getType(), srcRelationName );
 
-        CollectionInfo dstCollection = 
+        CollectionInfo dstCollection =
                 getDefaultSchema().getCollection( dstEntityRef.getType(), dstRelationName );
 
         Results results = null;
@@ -905,19 +907,21 @@ public class CpRelationManager implements RelationManager {
 
         headEntity = em.validate( headEntity );
 
-        CollectionInfo collection = 
+        CollectionInfo collection =
             getDefaultSchema().getCollection( headEntity.getType(), collName );
 
         if ( collection == null ) {
-            throw new RuntimeException( "Cannot find collection-info for '" + collName 
+            throw new RuntimeException( "Cannot find collection-info for '" + collName
                     + "' of " + headEntity.getType() + ":" + headEntity .getUuid() );
         }
 
-        IndexScope indexScope = new IndexScopeImpl(
-            cpHeadEntity.getId(), 
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
+        final IndexScope indexScope = new IndexScopeImpl(
+            cpHeadEntity.getId(),
+            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
 
-        EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+        final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+
+        final SearchTypes types = SearchTypes.fromTypes( collection.getType() );
 
         logger.debug( "Searching scope {}:{}",
 
@@ -938,9 +942,11 @@ public class CpRelationManager implements RelationManager {
 
         boolean satisfied = false;
 
+
+
         while ( !satisfied && queryCount++ < maxQueries ) {
 
-            CandidateResults crs = ei.search( indexScope, query );
+            CandidateResults crs = ei.search( indexScope, types, query );
 
             if ( results == null ) {
                 logger.debug( "Calling build results 1" );
@@ -993,8 +999,8 @@ public class CpRelationManager implements RelationManager {
 
         ConnectionRefImpl connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef );
 
-        CollectionScope targetScope = new CollectionScopeImpl( 
-            applicationScope.getApplication(), 
+        CollectionScope targetScope = new CollectionScopeImpl(
+            applicationScope.getApplication(),
             applicationScope.getApplication(),
             CpNamingUtils.getCollectionScopeNameFromEntityType( connectedEntityRef.getType() ) );
 
@@ -1003,27 +1009,27 @@ public class CpRelationManager implements RelationManager {
         if ( logger.isDebugEnabled() ) {
             logger.debug("createConnection(): "
                 + "Indexing connection type '{}'\n   from source {}:{}]\n"
-                + "   to target {}:{}\n   from scope\n   app {}\n   owner {}\n   name {}", 
+                + "   to target {}:{}\n   from scope\n   app {}\n   owner {}\n   name {}",
                 new Object[] {
                     connectionType,
                     headEntity.getType(),
                     headEntity.getUuid(),
                     connectedEntityRef.getType(),
                     connectedEntityRef.getUuid(),
-                    targetScope.getApplication(), 
-                    targetScope.getOwner(), 
+                    targetScope.getApplication(),
+                    targetScope.getOwner(),
                     targetScope.getName()
             });
         }
 
-        Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType()); 
-        org.apache.usergrid.persistence.model.entity.Entity targetEntity = 
+        Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType());
+        org.apache.usergrid.persistence.model.entity.Entity targetEntity =
             ((CpEntityManager)em).load( new CpEntityManager.EntityScope( targetScope, entityId));
 
         String edgeType = CpNamingUtils.getEdgeTypeFromConnectionType( connectionType );
 
         // create graph edge connection from head entity to member entity
-        Edge edge = new SimpleEdge( 
+        Edge edge = new SimpleEdge(
                 cpHeadEntity.getId(), edgeType, targetEntity.getId(), System.currentTimeMillis() );
 
         GraphManager gm = managerCache.getGraphManager( applicationScope );
@@ -1034,13 +1040,14 @@ public class CpRelationManager implements RelationManager {
 
         // Index the new connection in app|source|type context
         IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(),
-                CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ), entityType );
+                CpNamingUtils.getConnectionScopeName( connectionType ) );
 
         batch.index( indexScope, targetEntity );
 
         // Index the new connection in app|scope|all-types context
-        IndexScope allTypesIndexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.ALL_TYPES, entityType );
-        batch.index( allTypesIndexScope, targetEntity );
+        //TODO REMOVE INDEX CODE
+//        IndexScope allTypesIndexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.ALL_TYPES, entityType );
+//        batch.index( allTypesIndexScope, targetEntity );
 
 
         batch.execute();
@@ -1056,15 +1063,15 @@ public class CpRelationManager implements RelationManager {
 
     @SuppressWarnings( "unchecked" )
     @Metered( group = "core", name = "CpRelationManager_batchUpdateEntityConnection" )
-    public Mutator<ByteBuffer> batchUpdateEntityConnection( 
-            Mutator<ByteBuffer> batch, 
-            boolean disconnect, 
-            ConnectionRefImpl conn, 
+    public Mutator<ByteBuffer> batchUpdateEntityConnection(
+            Mutator<ByteBuffer> batch,
+            boolean disconnect,
+            ConnectionRefImpl conn,
             UUID timestampUuid ) throws Exception {
 
         long timestamp = getTimestampInMicros( timestampUuid );
 
-        Entity connectedEntity = em.get(new SimpleEntityRef( 
+        Entity connectedEntity = em.get(new SimpleEntityRef(
                 conn.getConnectedEntityType(), conn.getConnectedEntityId() ) );
 
         if ( connectedEntity == null ) {
@@ -1142,17 +1149,17 @@ public class CpRelationManager implements RelationManager {
 
         for ( String dictionaryName : dictionaryNames ) {
 
-            boolean has_dictionary = schema.hasDictionary( 
+            boolean has_dictionary = schema.hasDictionary(
                     connectedEntity.getType(), dictionaryName );
 
-            boolean dictionary_indexed = schema.isDictionaryIndexedInConnections( 
+            boolean dictionary_indexed = schema.isDictionaryIndexedInConnections(
                     connectedEntity.getType(), dictionaryName );
 
             if ( dictionary_indexed || !has_dictionary ) {
                 Set<Object> elementValues = em.getDictionaryAsSet( connectedEntity, dictionaryName );
                 for ( Object elementValue : elementValues ) {
-                    IndexUpdate indexUpdate = batchStartIndexUpdate( 
-                            batch, connectedEntity, dictionaryName, elementValue, 
+                    IndexUpdate indexUpdate = batchStartIndexUpdate(
+                            batch, connectedEntity, dictionaryName, elementValue,
                             timestampUuid, has_dictionary, true, disconnect, false );
                     batchUpdateConnectionIndex(indexUpdate, conn );
                 }
@@ -1165,7 +1172,7 @@ public class CpRelationManager implements RelationManager {
 
     @Override
     @Metered( group = "core", name = "RelationManager_createConnection_paired_connection_type" )
-    public ConnectionRef createConnection( 
+    public ConnectionRef createConnection(
             String pairedConnectionType,
             EntityRef pairedEntity,
             String connectionType,
@@ -1184,7 +1191,7 @@ public class CpRelationManager implements RelationManager {
 
 
     @Override
-    public ConnectionRef connectionRef( 
+    public ConnectionRef connectionRef(
             String connectionType, EntityRef connectedEntityRef ) throws Exception {
 
         ConnectionRef connection = new ConnectionRefImpl( headEntity, connectionType, connectedEntityRef );
@@ -1194,7 +1201,7 @@ public class CpRelationManager implements RelationManager {
 
 
     @Override
-    public ConnectionRef connectionRef( 
+    public ConnectionRef connectionRef(
             String pairedConnectionType,
             EntityRef pairedEntity,
             String connectionType,
@@ -1217,7 +1224,7 @@ public class CpRelationManager implements RelationManager {
         // First, clean up the dictionary records of the connection
         Keyspace ko = cass.getApplicationKeyspace( applicationId );
         Mutator<ByteBuffer> m = createMutator( ko, be );
-        batchUpdateEntityConnection( 
+        batchUpdateEntityConnection(
                 m, true, ( ConnectionRefImpl ) connectionRef, UUIDGenerator.newTimeUUID() );
         batchExecute( m, CassandraService.RETRY_COUNT );
 
@@ -1226,7 +1233,7 @@ public class CpRelationManager implements RelationManager {
 
         String connectionType = connectionRef.getConnectedEntity().getConnectionType();
 
-        CollectionScope targetScope = new CollectionScopeImpl( 
+        CollectionScope targetScope = new CollectionScopeImpl(
                 applicationScope.getApplication(),
                 applicationScope.getApplication(),
                 CpNamingUtils.getCollectionScopeNameFromEntityType( connectedEntityRef.getType()) );
@@ -1234,7 +1241,7 @@ public class CpRelationManager implements RelationManager {
         EntityCollectionManager targetEcm = managerCache.getEntityCollectionManager( targetScope );
 
         if ( logger.isDebugEnabled() ) {
-            logger.debug( "Deleting connection '{}' from source {}:{} \n   to target {}:{}", 
+            logger.debug( "Deleting connection '{}' from source {}:{} \n   to target {}:{}",
                 new Object[] {
                     connectionType,
                     connectingEntityRef.getType(),
@@ -1245,11 +1252,11 @@ public class CpRelationManager implements RelationManager {
         }
 
         Id entityId = new SimpleId( connectedEntityRef.getUuid(), connectedEntityRef.getType() );
-        org.apache.usergrid.persistence.model.entity.Entity targetEntity = 
+        org.apache.usergrid.persistence.model.entity.Entity targetEntity =
             ((CpEntityManager)em).load( new CpEntityManager.EntityScope( targetScope, entityId));
 
         // Delete graph edge connection from head entity to member entity
-        Edge edge = new SimpleEdge( 
+        Edge edge = new SimpleEdge(
             new SimpleId( connectingEntityRef.getUuid(),
                 connectingEntityRef.getType() ),
                 connectionType,
@@ -1263,20 +1270,20 @@ public class CpRelationManager implements RelationManager {
         final EntityIndexBatch batch = ei.createBatch();
 
         // Deindex the connection in app|source|type context
-        IndexScope indexScope = new IndexScopeImpl( 
+        IndexScope indexScope = new IndexScopeImpl(
             new SimpleId( connectingEntityRef.getUuid(),
                 connectingEntityRef.getType() ),
-                CpNamingUtils.getConnectionScopeName( targetEntity.getId().getType(),
-                connectionType ), entityType );
+                CpNamingUtils.getConnectionScopeName( connectionType ) );
         batch.deindex( indexScope, targetEntity );
 
         // Deindex the connection in app|source|type context
-        IndexScope allTypesIndexScope = new IndexScopeImpl( 
-            new SimpleId( connectingEntityRef.getUuid(),
-                connectingEntityRef.getType() ),
-                CpNamingUtils.ALL_TYPES, entityType );
-
-        batch.deindex( allTypesIndexScope, targetEntity );
+        //TODO REMOVE INDEX CODE
+//        IndexScope allTypesIndexScope = new IndexScopeImpl(
+//            new SimpleId( connectingEntityRef.getUuid(),
+//                connectingEntityRef.getType() ),
+//                CpNamingUtils.ALL_TYPES, entityType );
+//
+//        batch.deindex( allTypesIndexScope, targetEntity );
 
         batch.execute();
     }
@@ -1296,7 +1303,7 @@ public class CpRelationManager implements RelationManager {
 
     @Override
     public Set<String> getConnectionTypes( boolean filterConnection ) throws Exception {
-        Set<String> connections = cast( 
+        Set<String> connections = cast(
                 em.getDictionaryAsSet( headEntity, Schema.DICTIONARY_CONNECTED_TYPES ) );
 
         if ( connections == null ) {
@@ -1310,7 +1317,7 @@ public class CpRelationManager implements RelationManager {
 
 
     @Override
-    public Results getConnectedEntities( 
+    public Results getConnectedEntities(
             String connectionType, String connectedEntityType, Level level ) throws Exception {
 
         Results raw = null;
@@ -1326,15 +1333,11 @@ public class CpRelationManager implements RelationManager {
 
             headEntity = em.validate( headEntity );
 
-            String scopeName = null;
-            if ( connectedEntityType != null ) {
-                scopeName = CpNamingUtils.getConnectionScopeName( connectedEntityType, connectionType );
-            }
-            else {
-                scopeName = CpNamingUtils.ALL_TYPES;
-            }
 
-            IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), scopeName, entityType );
+            IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.getConnectionScopeName( connectionType ) );
+
+            final SearchTypes searchTypes = SearchTypes.fromNullableTypes( connectedEntityType );
+
 
             final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
 
@@ -1344,7 +1347,7 @@ public class CpRelationManager implements RelationManager {
                 indexScope.getName());
 
             query = adjustQuery( query );
-            CandidateResults crs = ei.search( indexScope, query );
+            CandidateResults crs = ei.search( indexScope, searchTypes,  query );
 
             raw = buildResults( query, crs, query.getConnectionType() );
         }
@@ -1369,7 +1372,7 @@ public class CpRelationManager implements RelationManager {
 
 
     @Override
-    public Results getConnectingEntities( 
+    public Results getConnectingEntities(
             String connType, String fromEntityType, Level resultsLevel ) throws Exception {
 
         return getConnectingEntities( connType, fromEntityType, resultsLevel, -1 );
@@ -1377,7 +1380,7 @@ public class CpRelationManager implements RelationManager {
 
 
     @Override
-    public Results getConnectingEntities( 
+    public Results getConnectingEntities(
             String connType, String fromEntityType, Level level, int count ) throws Exception {
 
         // looking for edges to the head entity
@@ -1418,37 +1421,19 @@ public class CpRelationManager implements RelationManager {
 
         headEntity = em.validate( headEntity );
 
-        if ( query.getEntityType() == null ) {
-
-            // search across all types of collections of the head-entity
-            IndexScope indexScope = new IndexScopeImpl( 
-                cpHeadEntity.getId(), 
-                CpNamingUtils.ALL_TYPES, entityType );
+        final IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(),
+                CpNamingUtils.getConnectionScopeName( query.getConnectionType() ) );
 
-            EntityIndex ei = managerCache.getEntityIndex( applicationScope );
+        final SearchTypes searchTypes = SearchTypes.fromNullableTypes( query.getEntityType() );
 
-            logger.debug("Searching connections from the all-types scope {}:{}",
-                indexScope.getOwner().toString(),
-                indexScope.getName());
-
-            query = adjustQuery( query );
-            CandidateResults crs = ei.search( indexScope, query );
-
-            return buildConnectionResults( query, crs, query.getConnectionType() );
-        }
-
-        IndexScope indexScope = new IndexScopeImpl( 
-            cpHeadEntity.getId(), 
-            CpNamingUtils.getConnectionScopeName( query.getEntityType(), 
-            query.getConnectionType() ), entityType );
         EntityIndex ei = managerCache.getEntityIndex( applicationScope );
-        
-        logger.debug("Searching connections from the scope {}:{}",
-            indexScope.getOwner().toString(),
-            indexScope.getName());
+
+        logger.debug( "Searching connections from the scope {}:{} with types {}", new Object[] {
+                        indexScope.getOwner().toString(), indexScope.getName(), searchTypes
+                } );
 
         query = adjustQuery( query );
-        CandidateResults crs = ei.search( indexScope, query );
+        CandidateResults crs = ei.search( indexScope, searchTypes, query );
 
         return buildConnectionResults( query, crs, query.getConnectionType() );
     }
@@ -1468,7 +1453,7 @@ public class CpRelationManager implements RelationManager {
                 // This is fulgy to put here, but required.
                 if ( query.getEntityType().equals( User.ENTITY_TYPE ) && ident.isEmail() ) {
 
-                    Query newQuery = Query.fromQL( "select * where email='" 
+                    Query newQuery = Query.fromQL( "select * where email='"
                             + query.getSingleNameOrEmailIdentifier() + "'" );
                     query.setRootOperand( newQuery.getRootOperand() );
                 }
@@ -1476,14 +1461,14 @@ public class CpRelationManager implements RelationManager {
                 // use the ident with the default alias. could be an email
                 else {
 
-                    Query newQuery = Query.fromQL( "select * where name='" 
+                    Query newQuery = Query.fromQL( "select * where name='"
                             + query.getSingleNameOrEmailIdentifier() + "'" );
                     query.setRootOperand( newQuery.getRootOperand() );
                 }
             }
             else if ( query.containsSingleUuidIdentifier() ) {
 
-                Query newQuery = Query.fromQL( 
+                Query newQuery = Query.fromQL(
                         "select * where uuid='" + query.getSingleUuidIdentifier() + "'" );
                 query.setRootOperand( newQuery.getRootOperand() );
             }
@@ -1491,7 +1476,7 @@ public class CpRelationManager implements RelationManager {
 
         if ( query.isReversed() ) {
 
-            Query.SortPredicate desc = 
+            Query.SortPredicate desc =
                 new Query.SortPredicate( PROPERTY_CREATED, Query.SortDirection.DESCENDING );
 
             try {
@@ -1504,7 +1489,7 @@ public class CpRelationManager implements RelationManager {
 
         if ( query.getSortPredicates().isEmpty() ) {
 
-            Query.SortPredicate asc = 
+            Query.SortPredicate asc =
                 new Query.SortPredicate( PROPERTY_CREATED, Query.SortDirection.ASCENDING);
 
             query.addSort( asc );
@@ -1541,7 +1526,7 @@ public class CpRelationManager implements RelationManager {
     }
 
 
-    private Results buildConnectionResults( 
+    private Results buildConnectionResults(
             Query query, CandidateResults crs, String connectionType ) {
 
         if ( query.getLevel().equals( Level.ALL_PROPERTIES ) ) {
@@ -1554,10 +1539,10 @@ public class CpRelationManager implements RelationManager {
 
         for ( CandidateResult cr : crs ) {
 
-            SimpleEntityRef targetRef = 
+            SimpleEntityRef targetRef =
                     new SimpleEntityRef( cr.getId().getType(), cr.getId().getUuid() );
 
-            final ConnectionRef ref = 
+            final ConnectionRef ref =
                     new ConnectionRefImpl( sourceRef, connectionType, targetRef );
 
             refs.add( ref );
@@ -1579,7 +1564,7 @@ public class CpRelationManager implements RelationManager {
         logger.debug( "buildResults() for {} from {} candidates", collName, crs.size() );
 
         //get an instance of our results loader
-        final ResultsLoader resultsLoader = this.resultsLoaderFactory.getLoader( 
+        final ResultsLoader resultsLoader = this.resultsLoaderFactory.getLoader(
                 applicationScope, this.headEntity, query.getResultsLevel() );
 
         //load the results
@@ -1607,7 +1592,7 @@ public class CpRelationManager implements RelationManager {
         elementValue = getDefaultSchema()
                 .validateEntitySetValue( entity.getType(), setName, elementValue );
 
-        IndexUpdate indexUpdate = batchStartIndexUpdate( batch, entity, setName, elementValue, 
+        IndexUpdate indexUpdate = batchStartIndexUpdate( batch, entity, setName, elementValue,
                 timestampUuid, true, true, removeFromSet, false );
 
         // Update collections 
@@ -1618,7 +1603,7 @@ public class CpRelationManager implements RelationManager {
         if ( containers != null ) {
             Map<EntityRef, Set<String>> containerEntities = getContainers();
             for ( EntityRef containerEntity : containerEntities.keySet() ) {
-                if ( containerEntity.getType().equals( TYPE_APPLICATION ) 
+                if ( containerEntity.getType().equals( TYPE_APPLICATION )
                         && Schema.isAssociatedEntityType( entity.getType() ) ) {
                     logger.debug( "Extended properties for {} not indexed by application", entity.getType() );
                     continue;
@@ -1645,15 +1630,15 @@ public class CpRelationManager implements RelationManager {
      * Batch update collection index.
      *
      * @param indexUpdate The update to apply
-     * @param owner The entity that is the owner context of this entity update. Can either be an 
+     * @param owner The entity that is the owner context of this entity update. Can either be an
      * application, or another * entity
      * @param collectionName the collection name
-     * 
+     *
      * @return The indexUpdate with batch mutations
      * @throws Exception the exception
      */
     @Metered( group = "core", name = "RelationManager_batchUpdateCollectionIndex" )
-    public IndexUpdate batchUpdateCollectionIndex( 
+    public IndexUpdate batchUpdateCollectionIndex(
             IndexUpdate indexUpdate, EntityRef owner, String collectionName )
             throws Exception {
 
@@ -1661,12 +1646,9 @@ public class CpRelationManager implements RelationManager {
 
         Entity indexedEntity = indexUpdate.getEntity();
 
-        String bucketId = indexBucketLocator.getBucket( 
-                applicationId, 
-                IndexBucketLocator.IndexType.COLLECTION, 
-                indexedEntity.getUuid(),
-                indexedEntity.getType(), 
-                indexUpdate.getEntryName() );
+        String bucketId = indexBucketLocator
+                .getBucket( applicationId, IndexBucketLocator.IndexType.COLLECTION, indexedEntity.getUuid(),
+                        indexedEntity.getType(), indexUpdate.getEntryName() );
 
         // the root name without the bucket
         // entity_id,collection_name,prop_name,
@@ -1684,13 +1666,13 @@ public class CpRelationManager implements RelationManager {
 
                 index_key = key( index_name, bucketId );
 
-                addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, index_key, 
+                addDeleteToMutator( indexUpdate.getBatch(), ENTITY_INDEX, index_key,
                         entry.getIndexComposite(), indexUpdate.getTimestamp() );
 
                 if ( "location.coordinates".equals( entry.getPath() ) ) {
-                    EntityLocationRef loc = new EntityLocationRef( indexUpdate.getEntity(), 
+                    EntityLocationRef loc = new EntityLocationRef( indexUpdate.getEntity(),
                             entry.getTimestampUuid(), entry.getValue().toString() );
-                    batchRemoveLocationFromCollectionIndex( indexUpdate.getBatch(), 
+                    batchRemoveLocationFromCollectionIndex( indexUpdate.getBatch(),
                             indexBucketLocator, applicationId, index_name, loc );
                 }
             }
@@ -1699,8 +1681,8 @@ public class CpRelationManager implements RelationManager {
             }
         }
 
-        if ( ( indexUpdate.getNewEntries().size() > 0 ) 
-                && ( !indexUpdate.isMultiValue() 
+        if ( ( indexUpdate.getNewEntries().size() > 0 )
+                && ( !indexUpdate.isMultiValue()
                 || ( indexUpdate.isMultiValue() && !indexUpdate.isRemoveListEntry() ) ) ) {
 
             for ( IndexUpdate.IndexEntry indexEntry : indexUpdate.getNewEntries() ) {
@@ -1713,15 +1695,15 @@ public class CpRelationManager implements RelationManager {
 
                 // int i = 0;
 
-                addInsertToMutator( indexUpdate.getBatch(), ENTITY_INDEX, index_key, 
+                addInsertToMutator( indexUpdate.getBatch(), ENTITY_INDEX, index_key,
                         indexEntry.getIndexComposite(), null, indexUpdate.getTimestamp() );
 
                 if ( "location.coordinates".equals( indexEntry.getPath() ) ) {
-                    EntityLocationRef loc = new EntityLocationRef( 
-                            indexUpdate.getEntity(), 
-                            indexEntry.getTimestampUuid(), 
+                    EntityLocationRef loc = new EntityLocationRef(
+                            indexUpdate.getEntity(),
+                            indexEntry.getTimestampUuid(),
                             indexEntry.getValue().toString() );
-                    batchStoreLocationInCollectionIndex( 
+                    batchStoreLocationInCollectionIndex(
                             indexUpdate.getBatch(),
                             indexBucketLocator,
                             applicationId,
@@ -1744,18 +1726,18 @@ public class CpRelationManager implements RelationManager {
     }
 
 
-    public IndexUpdate batchStartIndexUpdate( 
+    public IndexUpdate batchStartIndexUpdate(
             Mutator<ByteBuffer> batch, Entity entity, String entryName,
             Object entryValue, UUID timestampUuid, boolean schemaHasProperty,
              boolean isMultiValue, boolean removeListEntry, boolean fulltextIndexed )
             throws Exception {
-        return batchStartIndexUpdate( batch, entity, entryName, entryValue, timestampUuid, 
+        return batchStartIndexUpdate( batch, entity, entryName, entryValue, timestampUuid,
                 schemaHasProperty, isMultiValue, removeListEntry, fulltextIndexed, false );
     }
 
 
     @Metered(group = "core", name = "RelationManager_batchStartIndexUpdate")
-    public IndexUpdate batchStartIndexUpdate( 
+    public IndexUpdate batchStartIndexUpdate(
         Mutator<ByteBuffer> batch, Entity entity, String entryName,
         Object entryValue, UUID timestampUuid, boolean schemaHasProperty,
         boolean isMultiValue, boolean removeListEntry, boolean fulltextIndexed,
@@ -1763,7 +1745,7 @@ public class CpRelationManager implements RelationManager {
 
         long timestamp = getTimestampInMicros( timestampUuid );
 
-        IndexUpdate indexUpdate = new IndexUpdate( batch, entity, entryName, entryValue, 
+        IndexUpdate indexUpdate = new IndexUpdate( batch, entity, entryName, entryValue,
                 schemaHasProperty, isMultiValue, removeListEntry, timestampUuid );
 
         // entryName = entryName.toLowerCase();
@@ -1775,29 +1757,29 @@ public class CpRelationManager implements RelationManager {
             List<HColumn<ByteBuffer, ByteBuffer>> entries = null;
 
             if ( isMultiValue && validIndexableValue( entryValue ) ) {
-                entries = cass.getColumns( 
-                    cass.getApplicationKeyspace( applicationId ), 
+                entries = cass.getColumns(
+                    cass.getApplicationKeyspace( applicationId ),
                         ENTITY_INDEX_ENTRIES,
                         entity.getUuid(),
-                        new DynamicComposite( 
-                            entryName, 
-                            indexValueCode( entryValue ), 
+                        new DynamicComposite(
+                            entryName,
+                            indexValueCode( entryValue ),
                             toIndexableValue( entryValue ) ),
-                        setGreaterThanEqualityFlag( 
-                            new DynamicComposite( 
+                        setGreaterThanEqualityFlag(
+                            new DynamicComposite(
                                 entryName, indexValueCode( entryValue ),
-                                toIndexableValue( entryValue ) ) ), 
-                        INDEX_ENTRY_LIST_COUNT, 
+                                toIndexableValue( entryValue ) ) ),
+                        INDEX_ENTRY_LIST_COUNT,
                         false );
             }
             else {
-                entries = cass.getColumns( 
-                    cass.getApplicationKeyspace( applicationId ), 
-                    ENTITY_INDEX_ENTRIES, 
-                    entity.getUuid(), 
-                    new DynamicComposite( entryName ), 
-                    setGreaterThanEqualityFlag( new DynamicComposite( entryName ) ), 
-                    INDEX_ENTRY_LIST_COUNT, 
+                entries = cass.getColumns(
+                    cass.getApplicationKeyspace( applicationId ),
+                    ENTITY_INDEX_ENTRIES,
+                    entity.getUuid(),
+                    new DynamicComposite( entryName ),
+                    setGreaterThanEqualityFlag( new DynamicComposite( entryName ) ),
+                    INDEX_ENTRY_LIST_COUNT,
                     false );
             }
 
@@ -1816,7 +1798,7 @@ public class CpRelationManager implements RelationManager {
                 // new format:
                 // composite(entryName,
                 // value_code,prev_value,prev_timestamp,prev_obj_path) = null
-                DynamicComposite composite = 
+                DynamicComposite composite =
                         DynamicComposite.fromByteBuffer( entry.getName().duplicate() );
                 prev_value = composite.get( 2 );
                 prev_timestamp = ( UUID ) composite.get( 3 );
@@ -1831,7 +1813,7 @@ public class CpRelationManager implements RelationManager {
                         entryPath = entryName + "." + prev_obj_path;
                     }
 
-                    indexUpdate.addPrevEntry( 
+                    indexUpdate.addPrevEntry(
                             entryPath, prev_value, prev_timestamp, entry.getName().duplicate() );
 
                     // composite(property_value,connected_entity_id,entry_timestamp)
@@ -1847,7 +1829,7 @@ public class CpRelationManager implements RelationManager {
 
         if ( !isMultiValue || ( isMultiValue && !removeListEntry ) ) {
 
-            List<Map.Entry<String, Object>> list = 
+            List<Map.Entry<String, Object>> list =
                     IndexUtils.getKeyValueList( entryName, entryValue, fulltextIndexed );
 
             if ( entryName.equalsIgnoreCase( "location" ) && ( entryValue instanceof Map ) ) {
@@ -1862,15 +1844,15 @@ public class CpRelationManager implements RelationManager {
             for ( Map.Entry<String, Object> indexEntry : list ) {
 
                 if ( validIndexableValue( indexEntry.getValue() ) ) {
-                    indexUpdate.addNewEntry( 
+                    indexUpdate.addNewEntry(
                             indexEntry.getKey(), toIndexableValue( indexEntry.getValue() ) );
                 }
             }
 
             if ( isMultiValue ) {
                 addInsertToMutator( batch, ENTITY_INDEX_ENTRIES, entity.getUuid(),
-                        asList( entryName, 
-                            indexValueCode( entryValue ), 
+                        asList( entryName,
+                            indexValueCode( entryValue ),
                             toIndexableValue( entryValue ),
                             indexUpdate.getTimestampUuid() ),
                         null, timestamp );
@@ -1891,7 +1873,7 @@ public class CpRelationManager implements RelationManager {
                     byte code = indexValueCode( indexEntry.getValue() );
                     Object val = toIndexableValue( indexEntry.getValue() );
                     addInsertToMutator( batch, ENTITY_INDEX_ENTRIES, entity.getUuid(),
-                            asList( entryName, code, val, indexUpdate.getTimestampUuid(), name ), 
+                            asList( entryName, code, val, indexUpdate.getTimestampUuid(), name ),
                             null, timestamp );
 
                     indexUpdate.addIndex( indexEntry.getKey() );
@@ -1915,13 +1897,13 @@ public class CpRelationManager implements RelationManager {
      * @throws Exception the exception
      */
     @Metered(group = "core", name = "RelationManager_batchUpdateBackwardConnectionsDictionaryIndexes")
-    public IndexUpdate batchUpdateBackwardConnectionsDictionaryIndexes( 
+    public IndexUpdate batchUpdateBackwardConnectionsDictionaryIndexes(
             IndexUpdate indexUpdate ) throws Exception {
 
         logger.debug( "batchUpdateBackwardConnectionsListIndexes" );
 
         boolean entityHasDictionary = getDefaultSchema()
-                .isDictionaryIndexedInConnections( 
+                .isDictionaryIndexedInConnections(
                         indexUpdate.getEntity().getType(), indexUpdate.getEntryName() );
 
         if ( !entityHasDictionary ) {
@@ -1934,7 +1916,7 @@ public class CpRelationManager implements RelationManager {
 
 
     /**
-     * Search each reverse connection type in the graph for connections.  
+     * Search each reverse connection type in the graph for connections.
      * If one is found, update the index appropriately
      *
      * @param indexUpdate The index update to use
@@ -1951,7 +1933,7 @@ public class CpRelationManager implements RelationManager {
 
         for ( String connectionType : connectionTypes ) {
 
-            PagingResultsIterator itr = 
+            PagingResultsIterator itr =
                     getReversedConnectionsIterator( targetEntity, connectionType );
 
             for ( Object connection : itr ) {
@@ -1982,7 +1964,7 @@ public class CpRelationManager implements RelationManager {
      * @throws Exception the exception
      */
     @Metered(group = "core", name = "RelationManager_batchUpdateConnectionIndex")
-    public IndexUpdate batchUpdateConnectionIndex( 
+    public IndexUpdate batchUpdateConnectionIndex(
             IndexUpdate indexUpdate, ConnectionRefImpl connection ) throws Exception {
 
         logger.debug( "batchUpdateConnectionIndex" );
@@ -1999,10 +1981,10 @@ public class CpRelationManager implements RelationManager {
                 batchDeleteConnectionIndexEntries( indexUpdate, entry, connection, index_keys );
 
                 if ( "location.coordinates".equals( entry.getPath() ) ) {
-                    EntityLocationRef loc = 
+                    EntityLocationRef loc =
                         new EntityLocationRef( indexUpdate.getEntity(), entry.getTimestampUuid(),
                         entry.getValue().toString() );
-                    batchDeleteLocationInConnectionsIndex( 
+                    batchDeleteLocationInConnectionsIndex(
                         indexUpdate.getBatch(), indexBucketLocator, applicationId,
                         index_keys, entry.getPath(), loc );
                 }
@@ -2012,7 +1994,7 @@ public class CpRelationManager implements RelationManager {
             }
         }
 
-        if ( ( indexUpdate.getNewEntries().size() > 0 ) 
+        if ( ( indexUpdate.getNewEntries().size() > 0 )
                 && ( !indexUpdate.isMultiValue() || ( indexUpdate.isMultiValue()
                 && !indexUpdate.isRemoveListEntry() ) ) ) {
 
@@ -2022,11 +2004,11 @@ public class CpRelationManager implements RelationManager {
 
                 if ( "location.coordinates".equals( indexEntry.getPath() ) ) {
                     EntityLocationRef loc =
-                            new EntityLocationRef( 
+                            new EntityLocationRef(
                         indexUpdate.getEntity(),
                         indexEntry.getTimestampUuid(),
                         indexEntry.getValue().toString() );
-                    batchStoreLocationInConnectionsIndex( 
+                    batchStoreLocationInConnectionsIndex(
                             indexUpdate.getBatch(), indexBucketLocator, applicationId,
                             index_keys, indexEntry.getPath(), loc );
                 }
@@ -2058,10 +2040,10 @@ public class CpRelationManager implements RelationManager {
      *
      * @return connectionType The name of the edges to search
      */
-    private PagingResultsIterator getReversedConnectionsIterator( 
+    private PagingResultsIterator getReversedConnectionsIterator(
             EntityRef targetEntity, String connectionType ) throws Exception {
 
-        return new PagingResultsIterator( 
+        return new PagingResultsIterator(
                 getConnectingEntities( targetEntity, connectionType, null, Level.REFS ) );
     }
 
@@ -2074,7 +2056,7 @@ public class CpRelationManager implements RelationManager {
      * @param connectedEntityType The connected entity type, if not specified all types are returned
      * @param resultsLevel The results level to return
      */
-    private Results getConnectingEntities( 
+    private Results getConnectingEntities(
             EntityRef targetEntity, String connectionType, String connectedEntityType,
             Level resultsLevel ) throws Exception {
 
@@ -2091,16 +2073,16 @@ public class CpRelationManager implements RelationManager {
      * @param connectedEntityType The connected entity type, if not specified all types are returned
      * @param count result limit
      */
-    private Results getConnectingEntities( EntityRef targetEntity, String connectionType, 
+    private Results getConnectingEntities( EntityRef targetEntity, String connectionType,
             String connectedEntityType, Level level, int count) throws Exception {
 
         Query query = new Query();
         query.setResultsLevel( level );
         query.setLimit( count );
 
-        final ConnectionRefImpl connectionRef = new ConnectionRefImpl( 
+        final ConnectionRefImpl connectionRef = new ConnectionRefImpl(
                 new SimpleEntityRef( connectedEntityType, null ), connectionType, targetEntity );
-        final ConnectionResultsLoaderFactory factory = 
+        final ConnectionResultsLoaderFactory factory =
                 new ConnectionResultsLoaderFactory( connectionRef );
 
         QueryProcessorImpl qp = new QueryProcessorImpl( query, null, em, factory );
@@ -2111,7 +2093,7 @@ public class CpRelationManager implements RelationManager {
 
 
     @Metered( group = "core", name = "RelationManager_batchDeleteConnectionIndexEntries" )
-    public Mutator<ByteBuffer> batchDeleteConnectionIndexEntries( 
+    public Mutator<ByteBuffer> batchDeleteConnectionIndexEntries(
             IndexUpdate indexUpdate,
             IndexUpdate.IndexEntry entry,
             ConnectionRefImpl connection,
@@ -2172,9 +2154,9 @@ public class CpRelationManager implements RelationManager {
         logger.debug( "batchAddConnectionIndexEntries" );
 
         // entity_id,prop_name
-        Object property_index_key = key( index_keys[ConnectionRefImpl.ALL], 
+        Object property_index_key = key( index_keys[ConnectionRefImpl.ALL],
                 INDEX_CONNECTIONS, entry.getPath(),
-                indexBucketLocator.getBucket( applicationId, 
+                indexBucketLocator.getBucket( applicationId,
                         IndexBucketLocator.IndexType.CONNECTION, index_keys[ConnectionRefImpl.ALL],
                         entry.getPath() ) );
 
@@ -2192,7 +2174,7 @@ public class CpRelationManager implements RelationManager {
 
         // entity_id,connection_type,entity_type,prop_name
         Object connection_type_and_entity_type_prop_index_key =
-            key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], 
+            key( index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE],
                 INDEX_CONNECTIONS, entry.getPath(),
                         indexBucketLocator.getBucket( applicationId, IndexBucketLocator.IndexType.CONNECTION,
                                 index_keys[ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE], entry.getPath() ) );
@@ -2213,7 +2195,7 @@ public class CpRelationManager implements RelationManager {
             conn.getUuid(), indexUpdate.getTimestamp() );
 
         // composite(property_value,connected_entity_id,entry_timestamp)
-        addInsertToMutator( indexUpdate.getBatch(), ENTITY_INDEX, 
+        addInsertToMutator( indexUpdate.getBatch(), ENTITY_INDEX,
             connection_type_and_entity_type_prop_index_key,
             entry.getIndexComposite( conn.getConnectedEntityId() ), conn.getUuid(),
             indexUpdate.getTimestamp() );
@@ -2231,7 +2213,7 @@ public class CpRelationManager implements RelationManager {
 
         private final ConnectionRefImpl connection;
 
-        /** True if we should search from source->target edges.  
+        /** True if we should search from source->target edges.
          * False if we should search from target<-source edges */
         private final boolean outgoing;
 
@@ -2239,7 +2221,7 @@ public class CpRelationManager implements RelationManager {
         /**
          * @param queryProcessor They query processor to use
          * @param connection The connection refernce
-         * @param outgoing The direction to search.  True if we should search from source->target 
+         * @param outgoing The direction to search.  True if we should search from source->target
          * edges.  False if we * should search from target<-source edges
          */
         public SearchConnectionVisitor( QueryProcessorImpl queryProcessor, ConnectionRefImpl connection,
@@ -2258,7 +2240,7 @@ public class CpRelationManager implements RelationManager {
         @Override
         protected IndexScanner secondaryIndexScan( QueryNode node, QuerySlice slice ) throws Exception {
 
-            UUID id = ConnectionRefImpl.getIndexId( 
+            UUID id = ConnectionRefImpl.getIndexId(
                     ConnectionRefImpl.BY_CONNECTION_AND_ENTITY_TYPE,
                     headEntity,
                     connection.getConnectionType(),
@@ -2297,8 +2279,8 @@ public class CpRelationManager implements RelationManager {
 
             queryProcessor.applyCursorAndSort( slice );
 
-            GeoIterator itr = new GeoIterator( 
-                new ConnectionGeoSearch( em, indexBucketLocator, cass, connection.getIndexId() ), 
+            GeoIterator itr = new GeoIterator(
+                new ConnectionGeoSearch( em, indexBucketLocator, cass, connection.getIndexId() ),
                 query.getLimit(),
                 slice,
                 node.getPropertyName(),
@@ -2361,11 +2343,11 @@ public class CpRelationManager implements RelationManager {
 
             //we need to iterate all connection types
             else {
-                connectionTypes = new ConnectionTypesIterator( 
+                connectionTypes = new ConnectionTypesIterator(
                         cass, applicationId, entityIdToUse, outgoing, size );
             }
 
-            IndexScanner connectionScanner = new ConnectedIndexScanner( 
+            IndexScanner connectionScanner = new ConnectedIndexScanner(
                     cass,
                     dictionaryType,
                     applicationId,
@@ -2402,7 +2384,7 @@ public class CpRelationManager implements RelationManager {
 
         Object keyPrefix = key( indexKey, slice.getPropertyName() );
 
-        IndexScanner scanner = new IndexBucketScanner( 
+        IndexScanner scanner = new IndexBucketScanner(
                 cass,
                 indexBucketLocator,
                 ENTITY_INDEX,

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index c35cb33..3c6e96d 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -241,7 +241,7 @@ public class FilteringLoader implements ResultsLoader {
 
         IndexScope indexScope = new IndexScopeImpl( 
             ownerId,
-            CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()), entityType );
+            CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()) );
 
         batch.deindex( indexScope, candidateResult );
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index f2fce47..cf27203 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -1,4 +1,5 @@
-package org.apache.usergrid.corepersistence.util;/*
+package org.apache.usergrid.corepersistence.util;
+/*
  * Licensed to the Apache Software Foundation (ASF) under one
  * or more contributor license agreements.  See the NOTICE file
  * distributed with this work for additional information
@@ -27,11 +28,6 @@ import org.apache.usergrid.persistence.Schema;
 public class CpNamingUtils {
 
     /**
-     * Edge types for all types
-     */
-    public static final String ALL_TYPES = "zzzalltypeszzz";
-
-    /**
      * Edge types for collection suffix
      */
     public static final String EDGE_COLL_SUFFIX = "zzzcollzzz";
@@ -54,8 +50,8 @@ public class CpNamingUtils {
     }
 
 
-    public static String getConnectionScopeName( String entityType, String connectionType ) {
-        String csn = EDGE_CONN_SUFFIX + connectionType + entityType;
+    public static String getConnectionScopeName( String connectionType ) {
+        String csn = EDGE_CONN_SUFFIX + connectionType ;
         return csn.toLowerCase();
     }
 
@@ -65,7 +61,7 @@ public class CpNamingUtils {
     }
 
 
-    public  static boolean isConnectionEdgeType( String type ) {
+    public static boolean isConnectionEdgeType( String type ) {
         return type.startsWith( EDGE_CONN_SUFFIX );
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 4cd77e5..bbd19ce 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -37,6 +37,7 @@ import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexFactory;
 import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
 import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
@@ -73,7 +74,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         }});
         em.refreshIndex();
         
-        assertEquals( 1, queryCollectionCp("things", "select *").size() );
+        assertEquals( 1, queryCollectionCp("things", "thing", "select *").size() );
 
         org.apache.usergrid.persistence.model.entity.Entity cpEntity = getCpEntity(thing);
         UUID oldVersion = cpEntity.getVersion();
@@ -90,7 +91,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         assertTrue( "New version is greater than old", 
                 UUIDComparator.staticCompare( newVersion, oldVersion ) > 0 );
 
-        assertEquals( 2, queryCollectionCp("things", "select *").size() );
+        assertEquals( 2, queryCollectionCp("things", "thing", "select *").size() );
     }
 
 
@@ -121,7 +122,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         }
         em.refreshIndex();
 
-        CandidateResults crs = queryCollectionCp( "things", "select *");
+        CandidateResults crs = queryCollectionCp( "things", "thing", "select *");
         Assert.assertEquals( "Expect no stale candidates yet", numEntities, crs.size() );
 
         // update each one a bunch of times
@@ -152,7 +153,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         em.refreshIndex();
 
         // query Core Persistence directly for total number of result candidates
-        crs = queryCollectionCp("things", "select *");
+        crs = queryCollectionCp("things", "thing", "select *");
         Assert.assertEquals( "Expect stale candidates", numEntities * (numUpdates + 1), crs.size());
 
         // query EntityManager for results and page through them
@@ -197,7 +198,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
         Thread.sleep(600);
 
         // query for total number of result candidates = numEntities
-        crs = queryCollectionCp("things", "select *");
+        crs = queryCollectionCp("things", "thing", "select *");
         Assert.assertEquals( "Expect stale candidates de-indexed", numEntities, crs.size());
     }
 
@@ -228,7 +229,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
      * Go around EntityManager and execute query directly against Core Persistence.
      * Results may include stale index entries.
      */
-    private CandidateResults queryCollectionCp( String collName, String query ) {
+    private CandidateResults queryCollectionCp(final  String collName, final String type, final String query ) {
 
         EntityManager em = app.getEntityManager();
 
@@ -240,10 +241,10 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
 
         IndexScope is = new IndexScopeImpl(
             new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
-            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
+            CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ) );
         Query rcq = Query.fromQL(query);
         rcq.setLimit(10000); // no paging
 
-        return ei.search( is, rcq );
+        return ei.search( is, SearchTypes.fromTypes( type ), rcq );
     }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 584d65b..b3eace5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -20,6 +20,9 @@ package org.apache.usergrid.persistence.index;/*
  */
 
 
+import java.util.Arrays;
+
+
 /**
  * Class to encapsulate search types
  */
@@ -56,4 +59,53 @@ public class SearchTypes {
     public static SearchTypes allTypes(){
         return ALL_TYPES;
     }
+
+
+    /**
+     * Create a search type from a potentially nullable set of string.  If they are null, or empty, then allTypes is returned
+     * otherwise the type will be returned
+     * @param types
+     * @return
+     */
+    public static SearchTypes fromNullableTypes(final String... types){
+
+        if(types == null || types.length == 0){
+            return allTypes();
+        }
+
+        return fromTypes( types );
+    }
+
+
+    @Override
+    public boolean equals( final Object o ) {
+        if ( this == o ) {
+            return true;
+        }
+        if ( !( o instanceof SearchTypes ) ) {
+            return false;
+        }
+
+        final SearchTypes that = ( SearchTypes ) o;
+
+        if ( !Arrays.equals( types, that.types ) ) {
+            return false;
+        }
+
+        return true;
+    }
+
+
+    @Override
+    public int hashCode() {
+        return Arrays.hashCode( types );
+    }
+
+
+    @Override
+    public String toString() {
+        return "SearchTypes{" +
+                "types=" + Arrays.toString( types ) +
+                '}';
+    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/eef2b70b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
index 2521a38..c972851 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EsRunner.java
@@ -36,7 +36,7 @@ public class EsRunner extends ITRunner {
         ElasticSearchResource rule = new ElasticSearchResource();
 
         try {
-//            rule.before();
+            rule.before();
         }
         catch ( Throwable throwable ) {
             throw new RuntimeException( "Unable to start ES", throwable );


[22/27] incubator-usergrid git commit: Fixed GeoIT.

Posted by to...@apache.org.
Fixed GeoIT.

Cleaned up Query class

Harded search methods.  Input validation was not done to ensure we have minimum data to execute queries.


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/41647349
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/41647349
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/41647349

Branch: refs/heads/two-dot-o
Commit: 41647349fe9735108a8ae99ca95d9c4448df9d30
Parents: de457e9
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 11 14:42:15 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 11 14:42:15 2014 -0700

----------------------------------------------------------------------
 .../corepersistence/CpRelationManager.java      | 19 +++--
 .../corepersistence/util/CpNamingUtils.java     |  2 +-
 .../cassandra/RelationManagerImpl.java          | 18 ++---
 .../org/apache/usergrid/persistence/GeoIT.java  |  9 +--
 .../apache/usergrid/persistence/QueryTest.java  | 15 ++++
 stack/corepersistence/pom.xml                   |  2 +-
 .../usergrid/persistence/index/SearchTypes.java | 21 ++++-
 .../index/impl/EsEntityIndexImpl.java           | 26 ++++--
 .../usergrid/persistence/index/query/Query.java | 84 +++++++++++---------
 stack/pom.xml                                   |  7 --
 stack/test-utils/pom.xml                        |  5 --
 11 files changed, 125 insertions(+), 83 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 561c269..9a729b8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -106,6 +106,7 @@ import org.apache.usergrid.utils.IndexUtils;
 import org.apache.usergrid.utils.MapUtils;
 import org.apache.usergrid.utils.UUIDUtils;
 
+import com.google.common.base.Preconditions;
 import com.yammer.metrics.annotation.Metered;
 
 import me.prettyprint.hector.api.Keyspace;
@@ -418,7 +419,7 @@ public class CpRelationManager implements RelationManager {
                         }
                         else {
 
-                            String connName = CpNamingUtils.getCollectionName( edge.getType() );
+                            String connName = CpNamingUtils.getConnectionType( edge.getType() );
                             indexScope =
                                     new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(), sourceEntity.getType() ),
                                             CpNamingUtils.getConnectionScopeName( connName ) );
@@ -1432,14 +1433,20 @@ public class CpRelationManager implements RelationManager {
     @Override
     public Results searchConnectedEntities( Query query ) throws Exception {
 
-        if ( query == null ) {
-            query = new Query();
-        }
+        Preconditions.checkNotNull(query, "query cannot be null");
+
+        final String connection = query.getConnectionType();
+
+        Preconditions.checkNotNull( connection, "connection must be specified" );
+
+//        if ( query == null ) {
+//            query = new Query();
+//        }
 
         headEntity = em.validate( headEntity );
 
         final IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(),
-                CpNamingUtils.getConnectionScopeName( query.getConnectionType() ) );
+                CpNamingUtils.getConnectionScopeName( connection ) );
 
         final SearchTypes searchTypes = SearchTypes.fromNullableTypes( query.getEntityType() );
 
@@ -1452,7 +1459,7 @@ public class CpRelationManager implements RelationManager {
         query = adjustQuery( query );
         CandidateResults crs = ei.search( indexScope, searchTypes, query );
 
-        return buildConnectionResults( query, crs, query.getConnectionType() );
+        return buildConnectionResults( query, crs, connection );
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
index cf27203..9154752 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/util/CpNamingUtils.java
@@ -66,7 +66,7 @@ public class CpNamingUtils {
     }
 
 
-    static public String getConnectionType( String edgeType ) {
+    static public String  getConnectionType( String edgeType ) {
         String[] parts = edgeType.split( "\\|" );
         return parts[1];
     }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
index 53bef80..be84176 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/RelationManagerImpl.java
@@ -80,11 +80,9 @@ import org.apache.usergrid.persistence.schema.CollectionInfo;
 import org.apache.usergrid.utils.IndexUtils;
 import org.apache.usergrid.utils.MapUtils;
 
+import com.google.common.base.Preconditions;
 import com.yammer.metrics.annotation.Metered;
 
-import me.prettyprint.cassandra.serializers.ByteBufferSerializer;
-import me.prettyprint.cassandra.serializers.StringSerializer;
-import me.prettyprint.cassandra.serializers.UUIDSerializer;
 import me.prettyprint.hector.api.Keyspace;
 import me.prettyprint.hector.api.beans.DynamicComposite;
 import me.prettyprint.hector.api.beans.HColumn;
@@ -139,7 +137,7 @@ import static org.apache.usergrid.utils.InflectionUtils.singularize;
 import static org.apache.usergrid.utils.MapUtils.addMapSet;
 import static org.apache.usergrid.utils.UUIDUtils.getTimestampInMicros;
 import static org.apache.usergrid.utils.UUIDUtils.newTimeUUID;
-import static org.apache.usergrid.persistence.cassandra.Serializers.*;
+
 import org.apache.usergrid.persistence.entities.Application;
 import org.apache.usergrid.persistence.index.query.Query.Level;
 
@@ -2037,12 +2035,14 @@ public class RelationManagerImpl implements RelationManager {
     @Metered(group = "core", name = "RelationManager_searchConnectedEntities")
     public Results searchConnectedEntities( Query query ) throws Exception {
 
-        if ( query == null ) {
-            query = new Query();
-        }
+        Preconditions.checkNotNull(query, "Query must not be null");
+
+
+        final String connectedEntityType = query.getEntityType();
+        final String connectionType = query.getConnectionType();
 
-        String connectedEntityType = query.getEntityType();
-        String connectionType = query.getConnectionType();
+        Preconditions.checkNotNull( connectedEntityType, "entityType must not be null" );
+        Preconditions.checkNotNull( connectionType, "connectionType must not be null" );
 
         headEntity = em.validate( headEntity );
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
index 7e4493c..39e39b2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/GeoIT.java
@@ -200,11 +200,11 @@ public class GeoIT extends AbstractCoreIT {
         em.refreshIndex();
 
         emSearchResults = em.searchConnectedEntities( user, 
-                Query.fromQL( "location within 2000 of 37.776753, -122.407846" ) );
+                Query.fromQL( "location within 2000 of 37.776753, -122.407846" ).setConnectionType( "likes" ) );
         assertEquals( 1, emSearchResults.size() );
 
         emSearchResults = em.searchConnectedEntities( user, 
-                Query.fromQL( "location within 1000 of 37.776753, -122.407846" ) );
+                Query.fromQL( "location within 1000 of 37.776753, -122.407846" ).setConnectionType( "likes" ) );
         assertEquals( 0, emSearchResults.size() );
     }
 
@@ -380,10 +380,7 @@ public class GeoIT extends AbstractCoreIT {
     @Test
     public void testGeoWithIntersection() throws Exception {
 
-        UUID applicationId = setup.createApplication( "testOrganization", "testGeoWithIntersection" );
-        assertNotNull( applicationId );
-
-        EntityManager em = setup.getEmf().getEntityManager( applicationId );
+        EntityManager em = app.getEntityManager();
         assertNotNull( em );
 
         int size = 100;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
index 42489c3..e8cb7d4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/QueryTest.java
@@ -128,6 +128,21 @@ public class QueryTest {
         assertEquals( 5, ( ( LongLiteral ) equal.getLiteral() ).getValue().intValue() );
     }
 
+    @Test
+    public void withinDistanceCorrect(){
+        final Query query = Query.fromQL( "location within 2000 of 37.776753, -122.407846" );
+
+        WithinOperand withinOperand = ( WithinOperand ) query.getRootOperand();
+
+        final float distance = withinOperand.getDistance().getFloatValue();
+        final float lat = withinOperand.getLatitude().getFloatValue();
+        final float lon = withinOperand.getLongitude().getFloatValue();
+
+        assertEquals( 2000f, distance, 0f );
+        assertEquals( 37.776753f, lat, 0f );
+        assertEquals( -122.407846f, lon, 0f );
+    }
+
 
     @Test
     public void testCodeEquals() {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 9bd53fa..b44fca7 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -59,7 +59,7 @@ limitations under the License.
         <commons.collections.version>3.2.1</commons.collections.version>
         <commons.io.version>2.4</commons.io.version>
         <commons.lang.version>3.1</commons.lang.version>
-        <elasticsearch.version>1.3.2</elasticsearch.version>
+        <elasticsearch.version>1.4.0</elasticsearch.version>
         <fasterxml-uuid.version>3.1.3</fasterxml-uuid.version>
         <guava.version>15.0</guava.version>
         <guice.version>4.0-beta5</guice.version>

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index b3eace5..35b3a8b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -69,7 +69,7 @@ public class SearchTypes {
      */
     public static SearchTypes fromNullableTypes(final String... types){
 
-        if(types == null || types.length == 0){
+        if(isEmpty(types) ){
             return allTypes();
         }
 
@@ -77,6 +77,25 @@ public class SearchTypes {
     }
 
 
+    /**
+     * Return true if the array is empty, or it's elements contain a null
+     * @param input
+     * @return
+     */
+    private static boolean isEmpty(final String[] input){
+        if(input == null || input.length == 0){
+            return true;
+        }
+
+        for(int i = 0; i < input.length; i ++){
+            if(input[i] == null){
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     @Override
     public boolean equals( final Object o ) {
         if ( this == o ) {

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index cb56c7d..a2c3090 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -59,6 +59,7 @@ import org.apache.usergrid.persistence.index.EntityIndexBatch;
 import org.apache.usergrid.persistence.index.IndexFig;
 import org.apache.usergrid.persistence.index.IndexScope;
 import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
 import org.apache.usergrid.persistence.index.query.CandidateResult;
 import org.apache.usergrid.persistence.index.query.CandidateResults;
 import org.apache.usergrid.persistence.index.query.Query;
@@ -217,6 +218,12 @@ public class EsEntityIndexImpl implements EntityIndex {
                           .setTemplate( config.getIndexPrefix() + "*" ).addMapping( "_default_",
                         xcb ) // set mapping as the default for all types
                         .execute().actionGet();
+
+        if(!pitr.isAcknowledged()){
+            throw new IndexException( "Unable to create default mappings" );
+        }
+
+
     }
 
 
@@ -242,12 +249,6 @@ public class EsEntityIndexImpl implements EntityIndex {
                                                  .setScroll( cursorTimeout + "m" ).setQuery( qb );
 
 
-            if ( logger.isDebugEnabled() ) {
-                logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} limit {}", new Object[] {
-                        this.indexName, context, entityTypes, qb.toString().replace( "\n", " " ), query.getLimit()
-                } );
-            }
-
 
             final FilterBuilder fb = query.createFilterBuilder();
 
@@ -296,11 +297,20 @@ public class EsEntityIndexImpl implements EntityIndex {
                 logger.debug( "   Sort: {} order by {}", booleanFieldName, order.toString() );
             }
 
+
+
+            if ( logger.isDebugEnabled() ) {
+                logger.debug( "Searching index {}\n  scope{} \n type {}\n   query {} ", new Object[] {
+                        this.indexName, context, entityTypes, srb
+                } );
+            }
+
+
             try {
                 searchResponse = srb.execute().actionGet();
             }
             catch ( Throwable t ) {
-                logger.error( "Unable to communicate with elasticsearch" );
+                logger.error( "Unable to communicate with elasticsearch", t );
                 failureMonitor.fail( "Unable to execute batch", t );
                 throw t;
             }
@@ -325,7 +335,7 @@ public class EsEntityIndexImpl implements EntityIndex {
                 searchResponse = ssrb.execute().actionGet();
             }
             catch ( Throwable t ) {
-                logger.error( "Unable to communicate with elasticsearch" );
+                logger.error( "Unable to communicate with elasticsearch", t );
                 failureMonitor.fail( "Unable to execute batch", t );
                 throw t;
             }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 07bee44..a057b02 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -91,7 +91,7 @@ public class Query {
     private Map<String, String> selectAssignments = new LinkedHashMap<String, String>();
     private boolean mergeSelectResults = false;
     private Level level = Level.ALL_PROPERTIES;
-    private String connection;
+    private String connectionType;
     private List<String> permissions;
     private boolean reversed;
     private boolean reversedSet = false;
@@ -125,7 +125,7 @@ public class Query {
                     ? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
             mergeSelectResults = q.mergeSelectResults;
             //level = q.level;
-            connection = q.connection;
+            connectionType = q.connectionType;
             permissions = q.permissions != null ? new ArrayList<String>( q.permissions ) : null;
             reversed = q.reversed;
             reversedSet = q.reversedSet;
@@ -143,41 +143,47 @@ public class Query {
     }
 
 
-    public QueryBuilder createQueryBuilder(final String context) {
-
-
-        QueryBuilder queryBuilder;
-
-
-        /**
-         * Add our filter for context to our query for fast execution.  Fast because it utilizes bitsets
-         * internally. See this post for more detail.
-         * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
-         */
-
+    public QueryBuilder createQueryBuilder( final String context ) {
 
 
+        QueryBuilder queryBuilder = null;
 
 
         //we have a root operand.  Translate our AST into an ES search
         if ( getRootOperand() != null ) {
+            //In the case of geo only queries, this will return null into the query builder.  Once we start
+            //using tiles, we won't need this check any longer, since a geo query will return a tile query + post filter
             QueryVisitor v = new EsQueryVistor();
+
             try {
                 getRootOperand().visit( v );
-
-            } catch ( IndexException ex ) {
+            }
+            catch ( IndexException ex ) {
                 throw new RuntimeException( "Error building ElasticSearch query", ex );
             }
 
-            // TODO evaluate performance when it's an all query.  Do we need to put the context term first for performance?
-            queryBuilder = QueryBuilders.boolQuery().must(  v.getQueryBuilder() )
-                                        .must( QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
-        } 
+
+            queryBuilder = v.getQueryBuilder();
+        }
+
+
+        /**
+         * Add our filter for context to our query for fast execution.  Fast because it utilizes bitsets
+         * internally. See this post for more detail.
+         * http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+         */
+
+
+        // TODO evaluate performance when it's an all query.  Do we need to put the context term first for performance?
+        if ( queryBuilder != null ) {
+            queryBuilder = QueryBuilders.boolQuery().must( queryBuilder ).must( QueryBuilders
+                    .termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ) );
+        }
 
         //nothing was specified ensure we specify the context in the search
-		else {
+        else {
             queryBuilder = QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-		}
+        }
 
         return queryBuilder;
     }
@@ -283,7 +289,7 @@ public class Query {
         String ql = QueryUtils.queryStrFrom( params );
         String type = ListUtils.first( params.get( "type" ) );
         Boolean reversed = ListUtils.firstBoolean( params.get( "reversed" ) );
-        String connection = ListUtils.first( params.get( "connection" ) );
+        String connection = ListUtils.first( params.get( "connectionType" ) );
         UUID start = ListUtils.firstUuid( params.get( "start" ) );
         String cursor = ListUtils.first( params.get( "cursor" ) );
         Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
@@ -563,17 +569,6 @@ public class Query {
         this.type = type;
     }
 
-
-    public String getConnectionType() {
-        return connection;
-    }
-
-
-    public void setConnectionType( String connection ) {
-        this.connection = connection;
-    }
-
-
     public List<String> getPermissions() {
         return permissions;
     }
@@ -1373,18 +1368,29 @@ public class Query {
     }
 
 
-    public void setQl( String ql ) {
+    public Query setQl( String ql ) {
         this.ql = ql;
+        return this;
     }
 
 
-    public List<Identifier> getIdentifiers() {
-        return identifiers;
+    /**
+     * Get the connection type
+     * @return
+     */
+    public String getConnectionType() {
+        return connectionType;
     }
 
 
-    public String getConnection() {
-        return connection;
+    /**
+     * Set the connection type
+     * @param connection
+     * @return
+     */
+    public Query setConnectionType( final String connection ) {
+        this.connectionType = connection;
+        return this;
     }
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/pom.xml
----------------------------------------------------------------------
diff --git a/stack/pom.xml b/stack/pom.xml
index ff70076..5a7bdd3 100644
--- a/stack/pom.xml
+++ b/stack/pom.xml
@@ -115,7 +115,6 @@
     <tomcat-version>7.0.52</tomcat-version>
     <antlr.version>3.4</antlr.version>
     <tika.version>1.4</tika.version>
-    <elasticsearch.version>1.3.2</elasticsearch.version>
     <mockito.version>1.10.8</mockito.version>
 
     <usergrid.it.forkCount>3</usergrid.it.forkCount>
@@ -1428,12 +1427,6 @@
       </dependency>
 
       <dependency>
-        <groupId>org.elasticsearch</groupId>
-        <artifactId>elasticsearch</artifactId>
-        <version>${elasticsearch.version}</version>
-      </dependency>
-
-      <dependency>
         <groupId>com.relayrides</groupId>
         <artifactId>pushy</artifactId>
         <!-- The sha in the version is the git commit used in this build.  Check out the pushy source, then this commit to build the library locally -->

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/41647349/stack/test-utils/pom.xml
----------------------------------------------------------------------
diff --git a/stack/test-utils/pom.xml b/stack/test-utils/pom.xml
index f8e618d..96685f7 100644
--- a/stack/test-utils/pom.xml
+++ b/stack/test-utils/pom.xml
@@ -251,11 +251,6 @@
         </dependency>
 
         <dependency>
-            <groupId>org.elasticsearch</groupId>
-            <artifactId>elasticsearch</artifactId>
-        </dependency>
-
-        <dependency>
             <groupId>org.apache.tomcat.embed</groupId>
             <artifactId>tomcat-embed-core</artifactId>
         </dependency>


[23/27] incubator-usergrid git commit: Cleaned up obscelete external resources

Posted by to...@apache.org.
Cleaned up obscelete external resources


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/12b50286
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/12b50286
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/12b50286

Branch: refs/heads/two-dot-o
Commit: 12b502865fcd144a231ac3cf2d10b9f93c2248e8
Parents: 4164734
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 11 15:24:29 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 11 15:24:29 2014 -0700

----------------------------------------------------------------------
 .../src/test/resources/usergrid-test.properties |   2 +-
 .../usergrid/ConcurrentCoreIteratorITSuite.java |   2 +-
 .../org/apache/usergrid/CoreITSetupImpl.java    |   1 -
 .../java/org/apache/usergrid/CoreITSuite.java   |   2 +-
 .../batch/job/AbstractSchedulerRuntimeIT.java   |   6 +-
 .../core/astyanax/ColumnNameIteratorTest.java   |  21 ++--
 .../MultiKeyColumnNameIteratorTest.java         |  20 ++--
 .../astyanax/MultiRowColumnIteratorTest.java    |  20 ++--
 .../core/cassandra/CassandraRule.java           | 114 ------------------
 .../persistence/core/test/ITRunner.java         |  20 +---
 .../graph/CommittedGraphManagerIT.java          |   2 +-
 .../persistence/graph/GraphManagerLoadTest.java |   4 +-
 .../graph/GraphManagerShardConsistencyIT.java   |   6 -
 .../graph/GraphManagerShardingIT.java           |   4 +-
 .../graph/GraphManagerStressTest.java           |   4 +-
 .../graph/StorageGraphManagerIT.java            |   2 +-
 .../graph/guice/TestGraphModule.java            |   2 +-
 .../graph/impl/EdgeDeleteListenerTest.java      |   6 +-
 .../graph/impl/NodeDeleteListenerTest.java      |   6 +-
 .../graph/impl/stage/EdgeDeleteRepairTest.java  |   4 +-
 .../graph/impl/stage/EdgeMetaRepairTest.java    |   6 +-
 .../EdgeMetaDataSerializationProxyV2Test.java   |   7 +-
 .../EdgeMetaDataSerializationV1Test.java        |   4 +-
 .../EdgeMetaDataSerializationV2Test.java        |   4 +-
 .../EdgeSerializationChopTest.java              |   4 +-
 .../serialization/EdgeSerializationTest.java    |   7 +-
 .../serialization/NodeSerializationTest.java    |   4 +-
 .../PermanentSerializationTest.java             |   2 +-
 .../impl/shard/EdgeShardSerializationTest.java  |   9 +-
 .../NodeShardCounterSerializationTest.java      |   8 +-
 .../index/impl/CorePerformanceIT.java           |   4 -
 .../index/impl/ElasticSearchResource.java       | 118 +------------------
 .../impl/EntityConnectionIndexImplTest.java     |   4 -
 .../persistence/index/impl/EntityIndexTest.java |   6 +-
 .../usergrid/launcher/EmbeddedServerHelper.java |   4 +-
 35 files changed, 92 insertions(+), 347 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/config/src/test/resources/usergrid-test.properties
----------------------------------------------------------------------
diff --git a/stack/config/src/test/resources/usergrid-test.properties b/stack/config/src/test/resources/usergrid-test.properties
index cfa1f2f..5889625 100644
--- a/stack/config/src/test/resources/usergrid-test.properties
+++ b/stack/config/src/test/resources/usergrid-test.properties
@@ -59,7 +59,7 @@ cassandra.writecl=QUORUM
 cassandra.lock.keyspace=Locks
 
 elasticsearch.startup=external
-elasticsearch.cluster_name=test_cluster
+elasticsearch.cluster_name=usergrid
 
 #Properties to control the number of buckets in the index.
 usergrid.index.defaultbucketsize=20

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java b/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
index 273c6df..1312950 100644
--- a/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
+++ b/stack/core/src/test/java/org/apache/usergrid/ConcurrentCoreIteratorITSuite.java
@@ -79,5 +79,5 @@ public class ConcurrentCoreIteratorITSuite {
     public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( );
     
     @ClassRule
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
index fbff413..b3cc046 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSetupImpl.java
@@ -80,7 +80,6 @@ public class CoreITSetupImpl implements CoreITSetup {
      */
     protected void before( Description description ) throws Throwable {
         LOG.info( "Setting up for {}", description.getDisplayName() );
-        elasticSearchResource.startEs();
         initialize();
     }
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java b/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
index ccdcd98..ff06a80 100644
--- a/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
+++ b/stack/core/src/test/java/org/apache/usergrid/CoreITSuite.java
@@ -51,7 +51,7 @@ public class CoreITSuite {
     public static CassandraResource cassandraResource = CassandraResource.newWithAvailablePorts( );
 
     @ClassRule
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 
 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
index 5e7288b..cc31d2d 100644
--- a/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/batch/job/AbstractSchedulerRuntimeIT.java
@@ -29,6 +29,7 @@ import org.apache.usergrid.persistence.index.impl.ElasticSearchResource;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.ClassRule;
 
 
 /**
@@ -42,8 +43,11 @@ public class AbstractSchedulerRuntimeIT {
     protected static final String RUNLOOP_PROP = "usergrid.scheduler.job.interval";
     protected static final String FAIL_PROP = "usergrid.scheduler.job.maxfail";
 
+    @ClassRule
     public static CassandraResource cassandraResource = SchedulerITSuite.cassandraResource;
-    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource().startEs();
+
+    @ClassRule
+    public static ElasticSearchResource elasticSearchResource = new ElasticSearchResource();
 
 
     private TestJobListener listener = new TestJobListener();

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
index 2522f33..547c504 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/ColumnNameIteratorTest.java
@@ -24,14 +24,18 @@ package org.apache.usergrid.persistence.core.astyanax;
 
 import java.util.HashMap;
 
+import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.ClassRule;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.guice.TestCommonModule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
+import com.google.inject.Inject;
 import com.netflix.astyanax.Keyspace;
 import com.netflix.astyanax.MutationBatch;
 import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@@ -46,11 +50,14 @@ import com.netflix.astyanax.util.RangeBuilder;
 import static org.junit.Assert.assertEquals;
 
 
+
+@RunWith( ITRunner.class )
+@UseModules( TestCommonModule.class )
 public class ColumnNameIteratorTest {
 
 
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
+    @Inject
+    public CassandraFig cassandraFig;
 
     protected static Keyspace keyspace;
 
@@ -62,8 +69,8 @@ public class ColumnNameIteratorTest {
     protected static final boolean TRUE = true;
 
 
-    @BeforeClass
-    public static void setup() throws ConnectionException {
+    @Before
+    public void setup() throws ConnectionException {
 
 
         final CassandraConfig cassandraConfig = new CassandraConfig() {
@@ -87,7 +94,7 @@ public class ColumnNameIteratorTest {
 
 
         AstyanaxKeyspaceProvider astyanaxKeyspaceProvider =
-                new AstyanaxKeyspaceProvider( rule.getCassandraFig(), cassandraConfig );
+                new AstyanaxKeyspaceProvider( cassandraFig, cassandraConfig );
 
         keyspace = astyanaxKeyspaceProvider.get();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
index 61b9e54..a96db39 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiKeyColumnNameIteratorTest.java
@@ -26,14 +26,18 @@ import java.util.Arrays;
 import java.util.Comparator;
 import java.util.HashMap;
 
+import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.ClassRule;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.guice.TestCommonModule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
+import com.google.inject.Inject;
 import com.netflix.astyanax.Keyspace;
 import com.netflix.astyanax.MutationBatch;
 import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@@ -52,10 +56,12 @@ import rx.functions.Func1;
 import static org.junit.Assert.assertEquals;
 
 
+@RunWith( ITRunner.class )
+@UseModules( TestCommonModule.class )
 public class MultiKeyColumnNameIteratorTest {
 
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
+    @Inject
+    public CassandraFig cassandraFig;
 
     protected static Keyspace keyspace;
 
@@ -67,8 +73,8 @@ public class MultiKeyColumnNameIteratorTest {
     protected static final boolean TRUE = true;
 
 
-    @BeforeClass
-    public static void setup() throws ConnectionException {
+    @Before
+    public  void setup() throws ConnectionException {
 
 
         final CassandraConfig cassandraConfig = new CassandraConfig() {
@@ -92,7 +98,7 @@ public class MultiKeyColumnNameIteratorTest {
 
 
         AstyanaxKeyspaceProvider astyanaxKeyspaceProvider =
-                new AstyanaxKeyspaceProvider( rule.getCassandraFig(), cassandraConfig );
+                new AstyanaxKeyspaceProvider( cassandraFig, cassandraConfig );
 
         keyspace = astyanaxKeyspaceProvider.get();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
index 04039e4..4530d4b 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/astyanax/MultiRowColumnIteratorTest.java
@@ -28,14 +28,18 @@ import java.util.Comparator;
 import java.util.HashMap;
 import java.util.concurrent.CountDownLatch;
 
+import org.junit.Before;
 import org.junit.BeforeClass;
-import org.junit.ClassRule;
 import org.junit.Test;
+import org.junit.runner.RunWith;
 
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
+import org.apache.usergrid.persistence.core.guice.TestCommonModule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
 
+import com.google.inject.Inject;
 import com.netflix.astyanax.Keyspace;
 import com.netflix.astyanax.MutationBatch;
 import com.netflix.astyanax.connectionpool.exceptions.ConnectionException;
@@ -55,10 +59,12 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 
 
+@RunWith( ITRunner.class )
+@UseModules( TestCommonModule.class )
 public class MultiRowColumnIteratorTest {
 
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
+    @Inject
+    public CassandraFig cassandraFig;
 
     protected static Keyspace keyspace;
 
@@ -70,8 +76,8 @@ public class MultiRowColumnIteratorTest {
     protected static final boolean TRUE = true;
 
 
-    @BeforeClass
-    public static void setup() throws ConnectionException {
+    @Before
+    public void setup() throws ConnectionException {
 
 
         final CassandraConfig cassandraConfig = new CassandraConfig() {
@@ -95,7 +101,7 @@ public class MultiRowColumnIteratorTest {
 
 
         AstyanaxKeyspaceProvider astyanaxKeyspaceProvider =
-                new AstyanaxKeyspaceProvider( rule.getCassandraFig(), cassandraConfig );
+                new AstyanaxKeyspaceProvider( cassandraFig, cassandraConfig );
 
         keyspace = astyanaxKeyspaceProvider.get();
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
deleted file mode 100644
index 68a56e4..0000000
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/cassandra/CassandraRule.java
+++ /dev/null
@@ -1,114 +0,0 @@
-package org.apache.usergrid.persistence.core.cassandra;
-
-
-import com.google.common.io.Files;
-import com.google.inject.Guice;
-import com.google.inject.Injector;
-import com.netflix.astyanax.test.EmbeddedCassandra;
-import com.netflix.config.ConfigurationManager;
-
-import java.io.File;
-import java.io.IOException;
-import org.apache.cassandra.io.util.FileUtils;
-import org.apache.usergrid.persistence.core.astyanax.CassandraFig;
-import org.apache.usergrid.persistence.core.util.AvailablePortFinder;
-import org.safehaus.guicyfig.Env;
-import org.safehaus.guicyfig.EnvironResource;
-import org.safehaus.guicyfig.GuicyFigModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-
-/**
- * @TODO - I wanted this in the test module but unfortunately that will create a circular dep
- *         due to the inclusion of the MigrationManager
- */
-public class CassandraRule extends EnvironResource {
-    private static final Logger LOG = LoggerFactory.getLogger( CassandraRule.class );
-
-    private static final Object mutex = new Object();
-
-    private static EmbeddedCassandra cass;
-
-    private static boolean started = false;
-
-    private final CassandraFig cassandraFig;
-
-    static{
-        ConfigurationManager.getDeploymentContext().setDeploymentEnvironment( "UNIT" );
-        try {
-            ConfigurationManager.loadCascadedPropertiesFromResources( "usergrid" );
-        }
-        catch ( IOException e ) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    public CassandraRule() {
-        super( Env.UNIT );
-
-        Injector injector = Guice.createInjector( new GuicyFigModule( CassandraFig.class ) );
-        cassandraFig = injector.getInstance( CassandraFig.class );
-    }
-
-
-    /**
-     * Get the cassandra fig
-     * @return
-     */
-    public CassandraFig getCassandraFig(){
-        return cassandraFig;
-    }
-
-    @Override
-    public void before() throws Throwable {
-
-        if ( !cassandraFig.isEmbedded()) {
-            LOG.info("Using external Cassandra");
-            return;
-        }
-
-        if ( started ) {
-            return;
-        }
-
-        synchronized ( mutex ) {
-
-            //second into mutex
-            if(started){
-                return;
-            }
-
-            File dataDir = Files.createTempDir();
-            dataDir.deleteOnExit();
-
-            //cleanup before we run, shouldn't be necessary, but had the directory exist during JVM kill
-            if( dataDir.exists() ) {
-                FileUtils.deleteRecursive( dataDir );
-            }
-
-            try {
-                LOG.info( "Starting cassandra" );
-
-                cass = new EmbeddedCassandra( dataDir, "Usergrid", 9160,
-                        AvailablePortFinder.getNextAvailable() );
-                cass.start();
-
-                LOG.info( "Cassandra boostrapped" );
-
-                started = true;
-            }
-            catch ( IOException e ) {
-                throw new RuntimeException( "Unable to start cassandra", e );
-            }
-        }
-    }
-
-
-    @Override
-    protected void after() {
-
-        //TODO TN.  this should only really happen when we shut down
-//        cass.stop();
-    }
-}

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/test/ITRunner.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/test/ITRunner.java b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/test/ITRunner.java
index 1a800ad..82587fd 100644
--- a/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/test/ITRunner.java
+++ b/stack/corepersistence/common/src/test/java/org/apache/usergrid/persistence/core/test/ITRunner.java
@@ -27,8 +27,6 @@ import org.junit.runners.model.FrameworkMethod;
 import org.junit.runners.model.InitializationError;
 import org.junit.runners.model.Statement;
 
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
-
 import com.google.inject.AbstractModule;
 import com.google.inject.Guice;
 import com.google.inject.Injector;
@@ -36,25 +34,11 @@ import com.google.inject.Module;
 
 
 /**
- * Run jukito with cassandra
+ * Run guice and inject it into our tests
  */
-public class ITRunner extends BlockJUnit4ClassRunner {// extends JukitoRunner {
-
-    //this is fugly, but we have no other way to start cassandra before the jukito runner
-    static{
-      CassandraRule rule = new CassandraRule();
-
-        try {
-            rule.before();
-        }
-        catch ( Throwable throwable ) {
-            //super nasty, but we need to bail if this happens
-            throwable.printStackTrace();
-            System.exit( -1 );
-        }
+public class ITRunner extends BlockJUnit4ClassRunner {
 
 
-    }
 
     private Injector injector;
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
index fa366ad..b471119 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/CommittedGraphManagerIT.java
@@ -21,10 +21,10 @@
 package org.apache.usergrid.persistence.graph;
 
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.model.entity.Id;
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
index c312714..916bdaa 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerLoadTest.java
@@ -29,7 +29,6 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -41,9 +40,10 @@ import org.slf4j.LoggerFactory;
 import org.apache.commons.lang3.time.StopWatch;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
 import org.apache.usergrid.persistence.model.entity.Id;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
index a748c05..0f25537 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardConsistencyIT.java
@@ -41,13 +41,11 @@ import javax.annotation.Nullable;
 
 import org.junit.After;
 import org.junit.Before;
-import org.junit.ClassRule;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.migration.schema.MigrationException;
 import org.apache.usergrid.persistence.core.migration.schema.MigrationManager;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
@@ -85,10 +83,6 @@ import static org.junit.Assert.fail;
 public class GraphManagerShardConsistencyIT {
     private static final Logger log = LoggerFactory.getLogger( GraphManagerShardConsistencyIT.class );
 
-
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
-
     private static final MetricRegistry registry = new MetricRegistry();
 
     private static final Meter writeMeter = registry.meter( "writeThroughput" );

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
index e6231e0..dfda4b1 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerShardingIT.java
@@ -23,7 +23,6 @@ package org.apache.usergrid.persistence.graph;
 
 import java.util.concurrent.TimeoutException;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -31,9 +30,10 @@ import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.DirectedEdgeMeta;
 import org.apache.usergrid.persistence.graph.serialization.impl.shard.NodeShardApproximation;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
index 0ed0492..7b912a0 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/GraphManagerStressTest.java
@@ -23,7 +23,6 @@ import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -35,8 +34,9 @@ import org.slf4j.LoggerFactory;
 import org.apache.commons.lang3.time.StopWatch;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.impl.SimpleSearchByEdgeType;
 import org.apache.usergrid.persistence.model.entity.Id;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
index cb9726d..6f3d388 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/StorageGraphManagerIT.java
@@ -23,13 +23,13 @@ package org.apache.usergrid.persistence.graph;
 
 import java.util.concurrent.atomic.AtomicInteger;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.runner.RunWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.impl.GraphManagerImpl;
 import org.apache.usergrid.persistence.model.entity.Id;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
index bbcfa6e..87a0f6c 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/guice/TestGraphModule.java
@@ -19,8 +19,8 @@
 package org.apache.usergrid.persistence.graph.guice;
 
 
-import org.apache.usergrid.persistence.core.guice.TestModule;
 import org.apache.usergrid.persistence.core.guice.CommonModule;
+import org.apache.usergrid.persistence.core.guice.TestModule;
 
 
 /**

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
index 0a1fe52..f5e9dc8 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/EdgeDeleteListenerTest.java
@@ -24,8 +24,6 @@ package org.apache.usergrid.persistence.graph.impl;
 import java.util.Iterator;
 import java.util.UUID;
 
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -34,8 +32,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.GraphFig;
 import org.apache.usergrid.persistence.graph.GraphManagerFactory;
 import org.apache.usergrid.persistence.graph.MarkedEdge;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
index f325620..aa5ae5f 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/NodeDeleteListenerTest.java
@@ -22,8 +22,6 @@ package org.apache.usergrid.persistence.graph.impl;
 import java.util.Iterator;
 import java.util.UUID;
 
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Rule;
@@ -33,8 +31,10 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.Edge;
 import org.apache.usergrid.persistence.graph.GraphFig;
 import org.apache.usergrid.persistence.graph.GraphManager;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
index 1f300b1..d27aca7 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeDeleteRepairTest.java
@@ -22,7 +22,6 @@ package org.apache.usergrid.persistence.graph.impl.stage;
 
 import java.util.Iterator;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -31,8 +30,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.MarkedEdge;
 import org.apache.usergrid.persistence.graph.SearchByEdgeType;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
index f4711d3..156d1d0 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/impl/stage/EdgeMetaRepairTest.java
@@ -24,16 +24,16 @@ import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Set;
 
-import org.apache.usergrid.persistence.core.guice.ProxyImpl;
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.guice.ProxyImpl;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.GraphFig;
 import org.apache.usergrid.persistence.graph.MarkedEdge;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
index 2661c19..ea03b44 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationProxyV2Test.java
@@ -22,15 +22,14 @@
 package org.apache.usergrid.persistence.graph.serialization;
 
 
-import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
-import org.apache.usergrid.persistence.core.test.UseModules;
-
 import org.junit.After;
 import org.junit.Before;
 import org.junit.runner.RunWith;
 
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.guice.ProxyImpl;
+import org.apache.usergrid.persistence.core.migration.data.MigrationInfoSerialization;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationProxyImpl;
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
index 66a0902..068b27a 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV1Test.java
@@ -39,11 +39,11 @@ package org.apache.usergrid.persistence.graph.serialization;/*
  */
 
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.runner.RunWith;
 
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.guice.PreviousImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV1Impl;
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
index db41059..1fba055 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeMetaDataSerializationV2Test.java
@@ -22,11 +22,11 @@
 package org.apache.usergrid.persistence.graph.serialization;
 
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.runner.RunWith;
 
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.guice.CurrentImpl;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.serialization.impl.EdgeMetadataSerializationV2Impl;
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
index b4e5241..9dd362f 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationChopTest.java
@@ -22,15 +22,15 @@ package org.apache.usergrid.persistence.graph.serialization;
 import java.util.Iterator;
 import java.util.UUID;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.MarkedEdge;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.model.entity.Id;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
index 161df44..69d0a20 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/EdgeSerializationTest.java
@@ -26,7 +26,6 @@ import java.util.Set;
 import java.util.UUID;
 
 import org.junit.Before;
-import org.junit.ClassRule;
 import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
@@ -34,7 +33,6 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.graph.Edge;
 import org.apache.usergrid.persistence.graph.GraphFig;
@@ -72,14 +70,11 @@ import static org.mockito.Mockito.when;
  *
  *
  */
+
 public abstract class EdgeSerializationTest {
 
     private static final Logger log = LoggerFactory.getLogger( EdgeSerializationTest.class );
 
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
-
-
     @Inject
     @Rule
     public MigrationManagerRule migrationManagerRule;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
index 0736209..2391192 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/NodeSerializationTest.java
@@ -22,15 +22,15 @@ package org.apache.usergrid.persistence.graph.serialization;
 import java.util.Arrays;
 import java.util.Map;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/PermanentSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/PermanentSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/PermanentSerializationTest.java
index 9fb40ca..d850f76 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/PermanentSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/PermanentSerializationTest.java
@@ -20,10 +20,10 @@
 package org.apache.usergrid.persistence.graph.serialization;
 
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 
 import com.google.inject.Inject;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
index 133571a..cdac709 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/EdgeShardSerializationTest.java
@@ -22,17 +22,15 @@ package org.apache.usergrid.persistence.graph.serialization.impl.shard;
 
 import java.util.Iterator;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
-import org.junit.ClassRule;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.model.entity.Id;
 import org.apache.usergrid.persistence.model.util.UUIDGenerator;
@@ -53,9 +51,6 @@ import static org.mockito.Mockito.when;
 @UseModules({ TestGraphModule.class })
 public class EdgeShardSerializationTest {
 
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
-
 
     @Inject
     @Rule

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
index a7a281c..a443086 100644
--- a/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
+++ b/stack/corepersistence/graph/src/test/java/org/apache/usergrid/persistence/graph/serialization/impl/shard/count/NodeShardCounterSerializationTest.java
@@ -19,7 +19,6 @@
 package org.apache.usergrid.persistence.graph.serialization.impl.shard.count;
 
 
-import org.apache.usergrid.persistence.core.test.UseModules;
 import org.junit.Before;
 import org.junit.ClassRule;
 import org.junit.Rule;
@@ -29,9 +28,9 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
-import org.apache.usergrid.persistence.core.test.ITRunner;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.test.ITRunner;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.graph.GraphFig;
 import org.apache.usergrid.persistence.graph.guice.TestGraphModule;
 import org.apache.usergrid.persistence.graph.serialization.EdgeSerialization;
@@ -56,9 +55,6 @@ public class NodeShardCounterSerializationTest {
 
     private static final Logger log = LoggerFactory.getLogger( NodeShardCounterSerializationTest.class );
 
-    @ClassRule
-    public static CassandraRule rule = new CassandraRule();
-
 
     @Inject
     @Rule

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index 2f71e2e..6d09ccb 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -38,7 +38,6 @@ import org.apache.usergrid.persistence.collection.CollectionScope;
 import org.apache.usergrid.persistence.collection.EntityCollectionManager;
 import org.apache.usergrid.persistence.collection.EntityCollectionManagerFactory;
 import org.apache.usergrid.persistence.collection.impl.CollectionScopeImpl;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.index.EntityIndex;
@@ -72,9 +71,6 @@ public class CorePerformanceIT extends BaseIT {
     @ClassRule
     public static ElasticSearchResource es = new ElasticSearchResource();
 
-    @ClassRule
-    public static CassandraRule cass = new CassandraRule();
-
     // max entities we will write and read
     static int maxEntities = 10; // TODO: make this configurable when you add Chop 
 

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
index 661547a..1f04303 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/ElasticSearchResource.java
@@ -45,14 +45,10 @@ import com.google.inject.Injector;
 
 public class ElasticSearchResource extends EnvironResource {
 
-    private static final Logger log = LoggerFactory.getLogger( ElasticSearchResource.class );
-
-    private static Node node;
 
     private static int port;
     private static String host;
-    
-    private static boolean externalElasticSearch = false;
+
 
 
 
@@ -64,8 +60,6 @@ public class ElasticSearchResource extends EnvironResource {
             host=(String)props.getProperty( "elasticsearch.host", "127.0.0.1" );
             port=Integer.valueOf(props.getProperty( "elasticsearch.port", "9300" )).intValue();
             String forkString = props.getProperty("elasticsearch.startup", "external");
-            externalElasticSearch = "external".equals( forkString );
-
         } catch (Exception ex) {
             throw new RuntimeException("Error getting properties", ex);
         }
@@ -74,14 +68,8 @@ public class ElasticSearchResource extends EnvironResource {
 
     @Override
     protected void before() throws Throwable {
-    	if(externalElasticSearch){
             System.setProperty( IndexFig.ELASTICSEARCH_HOSTS, host );
-            System.setProperty( IndexFig.ELASTICSEARCH_PORT, port+"" );
-    		
-    	}else{
-            startEs();
-    	}
-    }
+            System.setProperty( IndexFig.ELASTICSEARCH_PORT, port+"" );    }
 
 
 
@@ -91,106 +79,4 @@ public class ElasticSearchResource extends EnvironResource {
     }
 
 
-    public synchronized ElasticSearchResource startEs(){
-
-//
-//        if ( node != null ) {
-//            return this;
-//        }
-//
-//
-//        //override the system properties in the Archiaus env
-//        port = AvailablePortFinder.getNextAvailable( 9300 );
-//
-//        final String host = "127.0.0.1";
-//        System.setProperty( IndexFig.ELASTICSEARCH_HOSTS, host );
-//        System.setProperty( IndexFig.ELASTICSEARCH_PORT, port + "" );
-//
-//        //we have to create this AFTER we set our system properties, or they won't get picked upt
-//        Injector injector = Guice.createInjector( new GuicyFigModule( IndexFig.class ) );
-//        IndexFig indexFig = injector.getInstance( IndexFig.class );
-//
-//
-//        final String clusterName = indexFig.getClusterName();
-//
-//        File tempDir;
-//        try {
-//            tempDir = getTempDirectory();
-//        }
-//        catch ( Exception ex ) {
-//            throw new RuntimeException( "Fatal error unable to create temp dir, start embedded ElasticSearch", ex );
-//        }
-//
-//
-//        Settings settings = ImmutableSettings.settingsBuilder()
-//                .put("cluster.name", clusterName)
-//                .put("network.publish_host", host)
-//                .put("transport.tcp.port", port)
-//                .put("discovery.zen.ping.multicast.enabled", "false")
-//                .put("node.http.enabled", false)
-//                .put("path.logs", tempDir.toString())
-//                .put("path.data", tempDir.toString())
-//                .put("index.store.type", "default")
-//                .put("index.number_of_shards", 1)
-//                .put("index.number_of_replicas", 1)
-//                .build();
-//
-//
-//        log.info( "-----------------------------------------------------------------------" );
-//        log.info( "Starting ElasticSearch embedded server settings: \n" + settings.getAsMap() );
-//        log.info( "-----------------------------------------------------------------------" );
-//
-//
-//        node = NodeBuilder.nodeBuilder().settings( settings ).clusterName( indexFig.getClusterName() ).client( false ).data( true ).build().start();
-//
-//        Runtime.getRuntime().addShutdownHook( new Thread() {
-//            @Override
-//            public void run() {
-//                shutdown();
-//            }
-//        } );
-//
-//        return this;
-        return this;
-    }
-
-
-    public static void shutdown() {
-    	if(!externalElasticSearch){
-            node.stop();
-    	}
-    }
-
-
-
-
-    /**
-     * Uses a project.properties file that Maven does substitution on to to replace the value of a property with the
-     * path to the Maven build directory (a.k.a. target). It then uses this path to generate a random String which it
-     * uses to append a path component to so a unique directory is selected. If already present it's deleted, then the
-     * directory is created.
-     *
-     * @return a unique temporary directory
-     *
-     * @throws java.io.IOException if we cannot access the properties file
-     */
-    public static File getTempDirectory() throws IOException {
-        File tmpdir;
-        Properties props = new Properties();
-        props.load( ClassLoader.getSystemResourceAsStream( "project.properties" ) );
-        File basedir = new File( ( String ) props.get( "target.directory" ) );
-        String comp = RandomStringUtils.randomAlphanumeric( 7 );
-        tmpdir = new File( basedir, comp );
-
-        if ( tmpdir.exists() ) {
-            log.info( "Deleting directory: {}", tmpdir );
-            FileUtils.forceDelete( tmpdir );
-        }
-        else {
-            log.info( "Creating temporary directory: {}", tmpdir );
-            FileUtils.forceMkdir( tmpdir );
-        }
-
-        return tmpdir;
-    }
 }

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index d2423bb..20dcbde 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -21,16 +21,12 @@ package org.apache.usergrid.persistence.index.impl;
 import java.io.IOException;
 import java.util.HashMap;
 
-import org.junit.ClassRule;
-import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
 import org.apache.usergrid.persistence.collection.util.EntityUtils;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
 import org.apache.usergrid.persistence.core.test.UseModules;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 72058ff..2b9ea62 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -25,9 +25,6 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 
-import org.apache.usergrid.persistence.core.test.UseModules;
-import org.junit.ClassRule;
-import org.junit.Rule;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.slf4j.Logger;
@@ -35,11 +32,10 @@ import org.slf4j.LoggerFactory;
 
 import org.apache.commons.lang3.time.StopWatch;
 
-import org.apache.usergrid.persistence.collection.guice.MigrationManagerRule;
 import org.apache.usergrid.persistence.collection.util.EntityUtils;
-import org.apache.usergrid.persistence.core.cassandra.CassandraRule;
 import org.apache.usergrid.persistence.core.scope.ApplicationScope;
 import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.core.test.UseModules;
 import org.apache.usergrid.persistence.core.util.Health;
 import org.apache.usergrid.persistence.index.EntityIndex;
 import org.apache.usergrid.persistence.index.EntityIndexBatch;

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/12b50286/stack/launcher/src/main/java/org/apache/usergrid/launcher/EmbeddedServerHelper.java
----------------------------------------------------------------------
diff --git a/stack/launcher/src/main/java/org/apache/usergrid/launcher/EmbeddedServerHelper.java b/stack/launcher/src/main/java/org/apache/usergrid/launcher/EmbeddedServerHelper.java
index c7669d8..b3b72c2 100644
--- a/stack/launcher/src/main/java/org/apache/usergrid/launcher/EmbeddedServerHelper.java
+++ b/stack/launcher/src/main/java/org/apache/usergrid/launcher/EmbeddedServerHelper.java
@@ -68,7 +68,7 @@ public class EmbeddedServerHelper {
 
             log.info( "Starting executor" );
 
-            executor.execute( new CassandraRunner() );
+            executor.execute( new ITRunner() );
             log.info( "Started executor" );
         }
         else {
@@ -180,7 +180,7 @@ public class EmbeddedServerHelper {
     }
 
 
-    class CassandraRunner implements Runnable {
+    class ITRunner implements Runnable {
 
         @Override
         public void run() {


[11/27] incubator-usergrid git commit: Changed prefix to match stack name

Posted by to...@apache.org.
Changed prefix to match stack name


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/33f74f55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/33f74f55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/33f74f55

Branch: refs/heads/two-dot-o
Commit: 33f74f5559ffabcc3ce14e506fc5594f230d4c66
Parents: 54772ab
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 12:57:13 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 12:57:13 2014 -0700

----------------------------------------------------------------------
 stack/awscluster/src/main/groovy/configure_usergrid.groovy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/33f74f55/stack/awscluster/src/main/groovy/configure_usergrid.groovy
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/groovy/configure_usergrid.groovy b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
index c42d84a..128d271 100644
--- a/stack/awscluster/src/main/groovy/configure_usergrid.groovy
+++ b/stack/awscluster/src/main/groovy/configure_usergrid.groovy
@@ -181,7 +181,7 @@ usergrid.user.resetpw.url=${baseUrl}/%s/%s/users/%s/resetpw
 
 usergrid.metrics.graphite.host=${graphite}
 
-usergrid.queue.prefix=${clusterName}
+usergrid.queue.prefix=${stackName}
 usergrid.queue.region=${ec2Region}
 
 """


[12/27] incubator-usergrid git commit: Updated yourkit

Posted by to...@apache.org.
Updated yourkit


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dd84f99e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dd84f99e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dd84f99e

Branch: refs/heads/two-dot-o
Commit: dd84f99ed8c316ea610d344da0bb90abf3d4a8ae
Parents: 33f74f5
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 14:27:06 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 14:27:06 2014 -0700

----------------------------------------------------------------------
 .../awscluster/src/main/dist/init_instance/install_yourkit.sh  | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dd84f99e/stack/awscluster/src/main/dist/init_instance/install_yourkit.sh
----------------------------------------------------------------------
diff --git a/stack/awscluster/src/main/dist/init_instance/install_yourkit.sh b/stack/awscluster/src/main/dist/init_instance/install_yourkit.sh
index 2883dae..ec1ac10 100644
--- a/stack/awscluster/src/main/dist/init_instance/install_yourkit.sh
+++ b/stack/awscluster/src/main/dist/init_instance/install_yourkit.sh
@@ -25,15 +25,15 @@ if [[ $YOURKIT = "true" ]]; then
 
 mkdir -p /mnt/yourkit
 cd /mnt/yourkit
-s3cmd --config=/etc/s3cfg get s3://${RELEASE_BUCKET}/yjp-2014-build-14108.zip
-unzip /mnt/yourkit/yjp-2014-build-14108.zip
+s3cmd --config=/etc/s3cfg get s3://${RELEASE_BUCKET}/yjp-2014-build-14110.zip
+unzip /mnt/yourkit/yjp-2014-build-14110.zip
 
 mkdir -p /mnt/yourkitreports
 
 chown -R tomcat7.tomcat7 /mnt/yourkitreports
 
 cat >> /etc/default/tomcat7 << EOF
-JAVA_OPTS="\${JAVA_OPTS} -agentpath:/mnt/yourkit/yjp-2014-build-14108/bin/linux-x86-64/libyjpagent.so=port=10001,logdir=/mnt/yourkitreports,dir=/mnt/yourkitreports,onexit=snapshot"
+JAVA_OPTS="\${JAVA_OPTS} -agentpath:/mnt/yourkit/yjp-2014-build-14110/bin/linux-x86-64/libyjpagent.so=port=10001,logdir=/mnt/yourkitreports,dir=/mnt/yourkitreports,onexit=snapshot"
 EOF
 
 fi


[21/27] incubator-usergrid git commit: Merge branch 'es-type-updates-ryan-external' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into es-type-updates-ryan-external

Posted by to...@apache.org.
Merge branch 'es-type-updates-ryan-external' of https://git-wip-us.apache.org/repos/asf/incubator-usergrid into es-type-updates-ryan-external


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/de457e90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/de457e90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/de457e90

Branch: refs/heads/two-dot-o
Commit: de457e9026b7c92c697c6cb026c8554f16ebab92
Parents: 8ae98f8 0fc8818
Author: Todd Nine <tn...@apigee.com>
Authored: Tue Nov 11 09:16:45 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue Nov 11 09:16:45 2014 -0700

----------------------------------------------------------------------
 .../apache/usergrid/rest/AbstractRestIT.java    |  22 +-
 .../apache/usergrid/rest/TestContextSetup.java  |  10 +-
 .../rest/management/OrganizationsIT.java        | 328 +++++++++----------
 .../usergrid/rest/test/security/TestUser.java   |   3 +
 4 files changed, 185 insertions(+), 178 deletions(-)
----------------------------------------------------------------------



[05/27] incubator-usergrid git commit: Merge remote-tracking branch 'ryan/APIBAAS-1362' into merge

Posted by to...@apache.org.
Merge remote-tracking branch 'ryan/APIBAAS-1362' into merge


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/ef9e94ec
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/ef9e94ec
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/ef9e94ec

Branch: refs/heads/two-dot-o
Commit: ef9e94ec75a688523c190bb84d4d5bd281eed310
Parents: eef2b70 bf2bd80
Author: Todd Nine <tn...@apigee.com>
Authored: Fri Nov 7 08:56:39 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Fri Nov 7 08:56:39 2014 -0700

----------------------------------------------------------------------
 .../index/impl/ElasticSearchResource.java       |  30 +++-
 stack/loadtests/README.md                       |  23 +++
 stack/loadtests/pom.xml                         |  10 --
 .../datagenerators/EntityDataGenerator.scala    |   5 +-
 .../org/apache/usergrid/helpers/Setup.scala     | 168 +++++++++++++++++++
 .../scenarios/NotificationScenarios.scala       |  11 +-
 .../usergrid/scenarios/UserScenarios.scala      |  15 +-
 .../org/apache/usergrid/settings/Settings.scala |  20 ++-
 .../usergrid/simulations/AppSimulation.scala    |  49 ++++++
 .../simulations/GeoProfileSimulation.scala      |  33 ++++
 .../PushNotificationTargetUserSimulation.scala  |  22 ++-
 stack/loadtests/src/test/scala/Engine.scala     |   2 +-
 .../usergrid/cassandra/CassandraResource.java   | 138 +++++++++++----
 13 files changed, 459 insertions(+), 67 deletions(-)
----------------------------------------------------------------------



[13/27] incubator-usergrid git commit: Merge remote-tracking branch 'origin/two-dot-o' into es-type-updates

Posted by to...@apache.org.
Merge remote-tracking branch 'origin/two-dot-o' into es-type-updates

Conflicts:
	stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java


Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/6ae1bef6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/6ae1bef6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/6ae1bef6

Branch: refs/heads/two-dot-o
Commit: 6ae1bef66e3061f1dae84e7913b6e72475c9090e
Parents: dd84f99 c282d80
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Nov 10 16:20:07 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Nov 10 16:20:07 2014 -0700

----------------------------------------------------------------------
 stack/core/pom.xml                              |   1 +
 .../corepersistence/CpEntityManager.java        |   5 +-
 .../corepersistence/CpRelationManager.java      |  22 ++-
 .../index/impl/ElasticSearchResource.java       |  30 +++-
 stack/loadtests/README.md                       |  27 +++
 stack/loadtests/pom.xml                         |  10 --
 .../datagenerators/EntityDataGenerator.scala    |  28 ++-
 .../datagenerators/FeederGenerator.scala        |  52 +++++-
 .../org/apache/usergrid/helpers/Setup.scala     | 178 +++++++++++++++++++
 .../scenarios/ConnectionScenarios.scala         |  29 ++-
 .../usergrid/scenarios/EntityScenarios.scala    |  65 +++++++
 .../scenarios/NotificationScenarios.scala       |  22 +--
 .../usergrid/scenarios/UserScenarios.scala      |  33 +++-
 .../org/apache/usergrid/settings/Settings.scala |  30 +++-
 .../usergrid/simulations/AppSimulation.scala    |  44 +++++
 .../simulations/ConnectionsSimulation.scala     |  53 ++++++
 .../PostCustomEntitySimulation.scala            |  61 +++++++
 .../simulations/PostDevicesSimulation.scala     |  45 -----
 .../PushNotificationTargetUserSimulation.scala  |  22 ++-
 .../usergrid/simulations/SetupSimulation.scala  |  45 +++++
 stack/loadtests/src/test/scala/Engine.scala     |   4 +-
 .../services/users/devices/DevicesService.java  |   4 +-
 .../usergrid/cassandra/CassandraResource.java   | 138 +++++++++++---
 23 files changed, 812 insertions(+), 136 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6ae1bef6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------

http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/6ae1bef6/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --cc stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index c822679,a47ce8c..20309a7
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@@ -689,8 -675,8 +707,10 @@@ public class CpRelationManager implemen
          //            headEntityScope.getName()});
  
          if ( connectBack && collection != null && collection.getLinkedCollection() != null ) {
+             getRelationManager( itemEntity ).addToCollection( 
+                     collection.getLinkedCollection(), headEntity, cpHeadEntity, false );
 +            getRelationManager( itemEntity ).addToCollection(
 +                    collection.getLinkedCollection(), headEntity, false );
          }
  
          return itemEntity;