You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2014/10/01 21:59:39 UTC
[1/2] git commit: Improvements to test, which now indexes data,
reads data, deletes application indexes,
rebuilds index and verifies that data is readable again.
Repository: incubator-usergrid
Updated Branches:
refs/heads/two-dot-o-rebuildable-index 352d69e02 -> 3ec144f3b
Improvements to test, which now indexes data, reads data, deletes application indexes, rebuilds index and verifies that data is readable again.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/df8c6b58
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/df8c6b58
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/df8c6b58
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: df8c6b585c03040e3ba2819ef76a2875eea8f982
Parents: 352d69e
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Oct 1 14:08:11 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Oct 1 14:08:11 2014 -0400
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 40 +++++-
.../corepersistence/CpEntityManagerFactory.java | 5 +-
.../corepersistence/CpRelationManager.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 124 ++++++++++++++++---
.../index/impl/EsEntityIndexImpl.java | 18 +++
5 files changed, 167 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 49be5c3..38453e1 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2907,12 +2907,27 @@ public class CpEntityManager implements EntityManager {
CollectionScope collScope = new CollectionScopeImpl(
applicationScope.getApplication(),
applicationScope.getApplication(),
- CpEntityManager.getCollectionScopeNameFromCollectionName(collName));
+ CpEntityManager.getCollectionScopeNameFromEntityType(entity.getType()));
EntityCollectionManager collMgr =
managerCache.getEntityCollectionManager(collScope);
org.apache.usergrid.persistence.model.entity.Entity collEntity =
- collMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+ collMgr.load( edge.getSourceNode() ).toBlockingObservable().last();
+
+// if (collEntity == null) {
+// if (logger.isDebugEnabled()) {
+// logger.error("FAILED to load entity {}:{} "
+// + "from scope\n app {}\n owner {}\n name {}",
+// new Object[]{
+// edge.getSourceNode().getType(),
+// edge.getSourceNode().getUuid(),
+// collScope.getApplication(),
+// collScope.getOwner(),
+// collScope.getName()
+// });
+// }
+// return;
+// }
CollectionScope memberScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -2924,6 +2939,21 @@ public class CpEntityManager implements EntityManager {
org.apache.usergrid.persistence.model.entity.Entity memberEntity =
memberMgr.load( edge.getTargetNode()).toBlockingObservable().last();
+// if (memberEntity == null) {
+// if (logger.isDebugEnabled()) {
+// logger.error("FAILED to load entity {}:{} "
+// + "from scope\n app {}\n owner {}\n name {}",
+// new Object[]{
+// edge.getTargetNode().getType(),
+// edge.getTargetNode().getUuid(),
+// memberScope.getApplication(),
+// memberScope.getOwner(),
+// memberScope.getName()
+// });
+// }
+// return;
+// }
+
indexEntityIntoCollections( collEntity, memberEntity, collName, true );
EntityRef ref = new SimpleEntityRef(
@@ -2948,7 +2978,7 @@ public class CpEntityManager implements EntityManager {
managerCache.getEntityCollectionManager(sourceScope);
org.apache.usergrid.persistence.model.entity.Entity sourceEntity =
- sourceEcm.load( edge.getTargetNode()).toBlockingObservable().last();
+ sourceEcm.load( fromEntityId ).toBlockingObservable().last();
CollectionScope targetScope = new CollectionScopeImpl(
applicationScope.getApplication(),
@@ -2998,6 +3028,10 @@ public class CpEntityManager implements EntityManager {
.getCollection( memberEntity.getId().getType(), collName);
if (connectBack && collection != null && collection.getLinkedCollection() != null) {
+
+ logger.debug("Linking back from entity in collection {} to collection {}",
+ collection.getName(), collection.getLinkedCollection());
+
indexEntityIntoCollections(
memberEntity, collectionEntity, collection.getLinkedCollection(), false );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 00a808c..b3d5135 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -597,11 +597,13 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
- logger.info("Rebuilding index for application id {}", appId);
EntityManager em = getEntityManager( appId );
Application app = em.getApplication();
((CpEntityManager)em).reindex( po );
+ em.refreshIndex();
+
+ logger.info("\n\nRebuilt index for application {} id {}\n", app.getName(), appId );
}
@@ -616,6 +618,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
@Override
public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po ) {
+ throw new UnsupportedOperationException( "Not supported yet." );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 04ed701..be21e85 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -916,7 +916,7 @@ public class CpRelationManager implements RelationManager {
results.merge( newResults );
}
- if ( crs.isEmpty() ) { // no more results
+ if ( crs.isEmpty() || !crs.hasCursor() ) { // no results, no cursor, can't get more
satisfied = true;
} else if ( results.size() == query.getLimit() ) { // got what we need
@@ -933,7 +933,7 @@ public class CpRelationManager implements RelationManager {
logger.warn("Satisfy query limit {}, new limit {} query count {}", new Object[] {
originalLimit, query.getLimit(), queryCount
});
- }
+ }
}
return results;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 7af66d3..663de59 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -33,11 +33,24 @@ import org.apache.usergrid.CoreApplication;
import com.codahale.metrics.Meter;
import com.codahale.metrics.MetricRegistry;
import com.codahale.metrics.Slf4jReporter;
+import com.google.inject.Injector;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.UUID;
import org.apache.usergrid.cassandra.Concurrent;
+import org.apache.usergrid.corepersistence.CpEntityManagerFactory;
+import org.apache.usergrid.corepersistence.CpSetup;
+import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.impl.EsEntityIndexImpl;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.model.entity.Id;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
@@ -52,7 +65,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
private static final long RUNTIME = TimeUnit.MINUTES.toMillis( 1 );
- private static final long writeDelayMs = 9;
+ private static final long writeDelayMs = 15;
//private static final long readDelayMs = 7;
@Rule
@@ -81,12 +94,15 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
@Test
- public void rebuildIndex() {
+ public void rebuildIndex() throws Exception {
logger.info("Started rebuildIndex()");
final EntityManager em = app.getEntityManager();
- final long stopTime = System.currentTimeMillis() + RUNTIME;
+
+ // ----------------- create a bunch of entities
+
+ final long stopTime = System.currentTimeMillis() + 300; // + RUNTIME;
final Map<String, Object> entityMap = new HashMap<>();
entityMap.put( "key1", 1000 );
@@ -94,29 +110,48 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
entityMap.put( "key3", "Some value" );
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
-
- int i = 0;
+ int entityCount = 0;
while ( System.currentTimeMillis() < stopTime ) {
-
- entityMap.put( "key", i );
+ entityMap.put("key", entityCount );
final Entity created;
try {
created = em.create("testType", entityMap );
} catch (Exception ex) {
throw new RuntimeException("Error creating entity", ex);
}
-
entityRefs.add( new SimpleEntityRef( created.getType(), created.getUuid() ) );
-
- if ( i % 100 == 0 ) {
- logger.info("Created {} entities", i );
+ if ( entityCount % 100 == 0 ) {
+ logger.info("Created {} entities", entityCount );
}
- i++;
-
+ entityCount++;
try { Thread.sleep( writeDelayMs ); } catch (InterruptedException ignored ) {}
}
- logger.info("Created {} entities", i);
+ logger.info("Created {} entities", entityCount);
+ em.refreshIndex();
+
+ // ----------------- test that we can read them, should work fine
+
+ logger.debug("Read the data");
+ readData("testTypes", entityCount );
+
+ // ----------------- delete the system and application indexes
+ logger.debug("Deleting app index and system app index");
+ deleteIndex( CpEntityManagerFactory.SYSTEM_APP_ID );
+ deleteIndex( em.getApplicationId() );
+
+ // ----------------- test that we can read them, should fail
+
+ logger.debug("Reading data, should fail this time ");
+ try {
+ readData( "testTypes", entityCount );
+ fail("should have failed to read data");
+
+ } catch (Exception expected) {}
+
+ // ----------------- rebuild index
+
+ logger.debug("Preparing to rebuild all indexes");;
final String meterName = this.getClass().getSimpleName() + ".rebuildIndex";
final Meter meter = registry.meter( meterName );
@@ -125,12 +160,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
int counter = 0;
@Override
public void onProgress( EntityRef s, EntityRef t, String etype ) {
-
meter.mark();
-
logger.debug("Indexing from {}:{} to {}:{} edgeType {}", new Object[] {
s.getType(), s.getUuid(), t.getType(), t.getUuid(), etype });
-
if ( !logger.isDebugEnabled() && counter % 100 == 0 ) {
logger.info("Reindexed {} entities", counter );
}
@@ -142,12 +174,68 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
setup.getEmf().rebuildAllIndexes( po );
registry.remove( meterName );
- logger.info("Finished rebuildIndex()");
+ logger.info("Rebuilt index");
} catch (Exception ex) {
logger.error("Error rebuilding index", ex);
fail();
}
+ // ----------------- test that we can read them
+
+ readData( "testTypes", entityCount );
+ }
+
+ /**
+ * Delete index for all applications, just need the one to get started.
+ */
+ private void deleteIndex( UUID appUuid ) {
+
+ Injector injector = CpSetup.getInjector();
+ EntityIndexFactory eif = injector.getInstance( EntityIndexFactory.class );
+
+ Id appId = new SimpleId( appUuid, "application");
+ IndexScope is = new IndexScopeImpl( appId, appId, "application");
+ EntityIndex ei = eif.createEntityIndex(is);
+ EsEntityIndexImpl eeii = (EsEntityIndexImpl)ei;
+
+ eeii.deleteIndex();
+ }
+
+ private int readData( String collectionName ) throws Exception {
+ return readData( collectionName, -1 );
+ }
+
+ private int readData( String collectionName, int expected ) throws Exception {
+
+ EntityManager em = app.getEntityManager();
+
+ Query q = Query.fromQL("select * where key1=1000");
+ q.setLimit(40);
+ Results results = em.searchCollection( em.getApplicationRef(), collectionName, q );
+
+ int count = 0;
+ while ( true ) {
+
+ for ( Entity e : results.getEntities() ) {
+ assertEquals( 2000, e.getProperty("key2"));
+ //if ( count % 100 == 0 ) {
+ logger.info( "read {} entities", count++);
+ //}
+ }
+
+ if ( results.hasCursor() ) {
+ logger.info( "Counted {} : query again with cursor", count);
+ q.setCursor( results.getCursor() );
+ results = em.searchCollection( em.getApplicationRef(), collectionName, q );
+ } else {
+ break;
+ }
+ }
+
+ if ( expected != -1 && expected != count ) {
+ throw new RuntimeException("Did not get expected " + expected + " entities");
+ }
+ return count;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/df8c6b58/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 8401e13..2eb8414 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -58,7 +58,10 @@ import org.apache.usergrid.persistence.model.field.SetField;
import org.apache.usergrid.persistence.model.field.StringField;
import org.apache.usergrid.persistence.model.field.UUIDField;
import org.apache.usergrid.persistence.model.field.value.EntityObject;
+import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequestBuilder;
+import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.index.IndexRequestBuilder;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
@@ -671,11 +674,13 @@ public class EsEntityIndexImpl implements EntityIndex {
}
+ @Override
public void refresh() {
client.admin().indices().prepareRefresh( indexName ).execute().actionGet();
log.debug("Refreshed index: " + indexName);
}
+
@Override
public CandidateResults getEntityVersions(Id id) {
Query query = new Query();
@@ -684,4 +689,17 @@ public class EsEntityIndexImpl implements EntityIndex {
return results;
}
+ /**
+ * For testing only.
+ */
+ public void deleteIndex() {
+ AdminClient adminClient = client.admin();
+ DeleteIndexResponse response = adminClient.indices().prepareDelete( indexName ).get();
+ if ( response.isAcknowledged() ) {
+ log.info("Deleted index: " + indexName );
+ } else {
+ log.info("Failed to delete index " + indexName );
+ }
+ }
+
}
[2/2] git commit: Couple of test fixes.
Posted by sn...@apache.org.
Couple of test fixes.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/3ec144f3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/3ec144f3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/3ec144f3
Branch: refs/heads/two-dot-o-rebuildable-index
Commit: 3ec144f3b5c7d76c225e9774d08b41de5f94ef54
Parents: df8c6b5
Author: Dave Johnson <dm...@apigee.com>
Authored: Wed Oct 1 15:57:37 2014 -0400
Committer: Dave Johnson <dm...@apigee.com>
Committed: Wed Oct 1 15:57:37 2014 -0400
----------------------------------------------------------------------
.../usergrid/persistence/PerformanceEntityRebuildIndexTest.java | 2 +-
.../usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ec144f3/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index 663de59..0f033a2 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -102,7 +102,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- create a bunch of entities
- final long stopTime = System.currentTimeMillis() + 300; // + RUNTIME;
+ final long stopTime = System.currentTimeMillis() + RUNTIME;
final Map<String, Object> entityMap = new HashMap<>();
entityMap.put( "key1", 1000 );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/3ec144f3/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
index efd9dfb..dc7ab3c 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImplIT.java
@@ -105,7 +105,7 @@ public class EntityManagerFactoryImplIT extends AbstractCoreIT {
traceTagManager.attach( traceTag );
logger.info( "EntityDaoTest.testCreateAndGet" );
- UUID applicationId = createApplication( "testOrganization", "testCreateAndGet" );
+ UUID applicationId = createApplication( "EntityManagerFactoryImplIT", "testCreateAndGet" );
logger.info( "Application id " + applicationId );
EntityManager em = emf.getEntityManager( applicationId );