You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/02 06:12:16 UTC
[07/12] git commit: Improvements to test, which now indexes data,
reads data, deletes application indexes,
rebuilds index and verifies that data is readable again.
Improvements to test, which now indexes data, reads data, deletes application indexes, rebuilds index and verifies that data is readable again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/df8c6b58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/df8c6b58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/df8c6b58
Branch: refs/heads/two-dot-o
Commit: df8c6b585c03040e3ba2819ef76a2875eea8f982
Parents: 352d69e
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Oct 1 14:08:11 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Oct 1 14:08:11 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 40 +++++-
.../corepersistence/CpEntityManagerFactory.java | 5 +-
.../corepersistence/CpRelationManager.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 124 ++++++++++++++++---
.../index/impl/EsEntityIndexImpl.java | 18 +++
5 files changed, 167 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 49be5c3..38453e1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2907,12 +2907,27 @@ public class CpEntityManager implements EntityManager {
CollectionScope collScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+ CpEntityManager.getCollectionScopeNameFromEntityType(entity.getType()));
EntityCollectionManager collMgr =
managerCache.getEntityCollectionManager(collScope);
org.apache.usergrid.persistence.model.entity.Entity collEntity =
- collMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+ collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
+
+// if (collEntity == null) {
+// if (logger.isDebugEnabled()) {
+// logger.error("FAILED to load entity {}:{} "
+// + "from scope\n app {}\n owner {}\n name {}",
+// new Object[]{
+// edge.getSourceNode().getType(),
+// edge.getSourceNode().getUuid(),
+// collScope.getApplication(),
+// collScope.getOwner(),
+// collScope.getName()
+// });
+// }
+// return;
+// }
CollectionScope memberScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -2924,6 +2939,21 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity memberEntity =
memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+// if (memberEntity == null) {
+// if (logger.isDebugEnabled()) {
+// logger.error("FAILED to load entity {}:{} "
+// + "from scope\n app {}\n owner {}\n name {}",
+// new Object[]{
+// edge.getTargetNode().getType(),
+// edge.getTargetNode().getUuid(),
+// memberScope.getApplication(),
+// memberScope.getOwner(),
+// memberScope.getName()
+// });
+// }
+// return;
+// }
+
indexEntityIntoCollections( collEntity, memberEntity, collName, true );
EntityRef ref = new SimpleEntityRef(
@@ -2948,7 +2978,7 @@ public class CpEntityManager implements EntityManager {
managerCache.getEntityCollectionManager(sourceScope);
org.apache.usergrid.persistence.model.entity.Entity sourceEntity =
- sourceEcm.load( edge.getTargetNode()).toBlockingObservable().last();
+ sourceEcm.load( fromEntityId ).toBlockingObservable().last();
CollectionScope targetScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -2998,6 +3028,10 @@ public class CpEntityManager implements EntityManager {
.getCollection( memberEntity.getId().getType(), collName);
if (connectBack && collection != null && collection.getLinkedCollection() != null) {
+
+ logger.debug("Linking back from entity in collection {} to collection {}",
+ collection.getName(), collection.getLinkedCollection());
+
indexEntityIntoCollections(
memberEntity, collectionEntity, collection.getLinkedCollection(), false );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 00a808c..b3d5135 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -597,11 +597,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
- logger.info("Rebuilding index for application id {}", appId);
EntityManager em = getEntityManager( appId );
Application app = em.getApplication();
((CpEntityManager)em).reindex( po );
+ em.refreshIndex();
+
+ logger.info("\n\nRebuilt index for application {} id {}\n", app.getName(), appId );
}
@@ -616,6 +618,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
+ throw new UnsupportedOperationException( "Not supported yet." );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 04ed701..be21e85 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -916,7 +916,7 @@ public class CpRelationManager implements RelationManager {
results.merge( newResults );
}
- if ( crs.isEmpty() ) { // no more results
+ if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
satisfied = true;
} else if ( results.size() == query.getLimit() ) { // got what we need
@@ -933,7 +933,7 @@ public class CpRelationManager implements RelationManager {
logger.warn("Satisfy query limit {}, new limit {} query count {}", new Object[] {
originalLimit, query.getLimit(), queryCount
});
- }
+ }
}
return results;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 7af66d3..663de59 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -33,11 +33,24 @@ import org.apache.usergrid.CoreApplication;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Slf4jReporter;
+import com.google.inject.Injector;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -52,7 +65,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
- private static final long writeDelayMs = 9;
+ private static final long writeDelayMs = 15;
//private static final long readDelayMs = 7;
@Rule
@@ -81,12 +94,15 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
@Test
- public void rebuildIndex() {
+ public void rebuildIndex() throws Exception {
logger.info("Started rebuildIndex()");
final EntityManager em = app.getEntityManager();
- final long stopTime = System.currentTimeMillis() + RUNTIME;
+
+ // ----------------- create a bunch of entities
+
+ final long stopTime = System.currentTimeMillis() + 300; // + RUNTIME;
final Map<String, Object> entityMap = new HashMap<>();
entityMap.put( "key1", 1000 );
@@ -94,29 +110,48 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
entityMap.put( "key3", "Some value" );
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
-
- int i = 0;
+ int entityCount = 0;
while ( System.currentTimeMillis() < stopTime ) {
-
- entityMap.put( "key", i );
+ entityMap.put("key", entityCount );
final Entity created;
try {
created = em.create("testType", entityMap );
} catch (Exception ex) {
throw new RuntimeException("Error creating entity", ex);
}
-
entityRefs.add( new SimpleEntityRef( created.getType(), created.getUuid() ) );
-
- if ( i % 100 == 0 ) {
- logger.info("Created {} entities", i );
+ if ( entityCount % 100 == 0 ) {
+ logger.info("Created {} entities", entityCount );
}
- i++;
-
+ entityCount++;
try { Thread.sleep( writeDelayMs ); } catch (InterruptedException ignored ) {}
}
- logger.info("Created {} entities", i);
+ logger.info("Created {} entities", entityCount);
+ em.refreshIndex();
+
+ // ----------------- test that we can read them, should work fine
+
+ logger.debug("Read the data");
+ readData("testTypes", entityCount );
+
+ // ----------------- delete the system and application indexes
+ logger.debug("Deleting app index and system app index");
+ deleteIndex( CpEntityManagerFactory.SYSTEM_APP_ID );
+ deleteIndex( em.getApplicationId() );
+
+ // ----------------- test that we can read them, should fail
+
+ logger.debug("Reading data, should fail this time ");
+ try {
+ readData( "testTypes", entityCount );
+ fail("should have failed to read data");
+
+ } catch (Exception expected) {}
+
+ // ----------------- rebuild index
+
+ logger.debug("Preparing to rebuild all indexes");;
final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
final Meter meter = registry.meter( meterName );
@@ -125,12 +160,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
int counter = 0;
@Override
public void onProgress( EntityRef s, EntityRef t, String etype ) {
-
meter.mark();
-
logger.debug("Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
-
if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
logger.info("Reindexed {} entities", counter );
}
@@ -142,12 +174,68 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
setup.getEmf().rebuildAllIndexes( po );
registry.remove( meterName );
- logger.info("Finished rebuildIndex()");
+ logger.info("Rebuilt index");
} catch (Exception ex) {
logger.error("Error rebuilding index", ex);
fail();
}
+ // ----------------- test that we can read them
+
+ readData( "testTypes", entityCount );
+ }
+
+ /**
+ * Delete index for all applications, just need the one to get started.
+ */
+ private void deleteIndex( UUID appUuid ) {
+
+ Injector injector = CpSetup.getInjector();
+ EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
+
+ Id appId = new SimpleId( appUuid, "application");
+ IndexScope is = new IndexScopeImpl( appId, appId, "application");
+ EntityIndex ei = eif.createEntityIndex(is);
+ EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
+
+ eeii.deleteIndex();
+ }
+
+ private int readData( String collectionName ) throws Exception {
+ return readData( collectionName, -1 );
+ }
+
+ private int readData( String collectionName, int expected ) throws Exception {
+
+ EntityManager em = app.getEntityManager();
+
+ Query q = Query.fromQL("select * where key1=1000");
+ q.setLimit(40);
+ Results results = em.searchCollection( em.getApplicationRef(), collectionName, q );
+
+ int count = 0;
+ while ( true ) {
+
+ for ( Entity e : results.getEntities() ) {
+ assertEquals( 2000, e.getProperty("key2"));
+ //if ( count % 100 == 0 ) {
+ logger.info( "read {} entities", count++);
+ //}
+ }
+
+ if ( results.hasCursor() ) {
+ logger.info( "Counted {} : query again with cursor", count);
+ q.setCursor( results.getCursor() );
+ results = em.searchCollection( em.getApplicationRef(), collectionName, q );
+ } else {
+ break;
+ }
+ }
+
+ if ( expected != -1 && expected != count ) {
+ throw new RuntimeException("Did not get expected " + expected + " entities");
+ }
+ return count;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8401e13..2eb8414 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -58,7 +58,10 @@ import org.apache.usergrid.persistence.model.field.SetField;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@@ -671,11 +674,13 @@ public class EsEntityIndexImpl implements EntityIndex {
}
+ @Override
public void refresh() {
client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
log.debug("Refreshed index: " + indexName);
}
+
@Override
public CandidateResults getEntityVersions(Id id) {
Query query = new Query();
@@ -684,4 +689,17 @@ public class EsEntityIndexImpl implements EntityIndex {
return results;
}
+ /**
+ * For testing only.
+ */
+ public void deleteIndex() {
+ AdminClient adminClient = client.admin();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+ if ( response.isAcknowledged() ) {
+ log.info("Deleted index: " + indexName );
+ } else {
+ log.info("Failed to delete index " + indexName );
+ }
+ }
+
}