You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sn...@apache.org on 2015/02/26 17:27:31 UTC
[07/50] [abbrv] incubator-usergrid git commit: reindexCollection now
implemented w/test in the Core module. Still needs a REST level test.
reindexCollection now implemented w/test in the Core module. Still needs a REST level test.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/2f077f1b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/2f077f1b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/2f077f1b
Branch: refs/heads/two-dot-o
Commit: 2f077f1bada2cb79db5dc9af19f101b668f92b9c
Parents: a704ed7
Author: Dave Johnson <dm...@apigee.com>
Authored: Fri Feb 20 13:52:21 2015 -0500
Committer: Dave Johnson <dm...@apigee.com>
Committed: Fri Feb 20 13:52:21 2015 -0500
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 37 ++++----
.../usergrid/corepersistence/CpWalker.java | 20 +++-
.../usergrid/persistence/EntityManager.java | 2 +-
.../cassandra/EntityManagerFactoryImpl.java | 11 +--
.../cassandra/EntityManagerImpl.java | 4 +-
.../PerformanceEntityRebuildIndexTest.java | 97 ++++++++++++--------
.../management/importer/ImportCollectionIT.java | 6 +-
7 files changed, 105 insertions(+), 72 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index 6af0539..721ac80 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -2745,11 +2745,12 @@ public class CpEntityManager implements EntityManager {
*/
@Override
public void reindexCollection(
- final EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception {
+ final EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception {
CpWalker walker = new CpWalker( );
- walker.walkCollections( this, application, collectionName, new CpVisitor() {
+ walker.walkCollections(
+ this, application, collectionName, reverse, new CpVisitor() {
@Override
public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
@@ -2759,9 +2760,9 @@ public class CpEntityManager implements EntityManager {
po.onProgress( entity );
}
catch ( WriteOptimisticVerifyException wo ) {
- //swallow this, it just means this was already updated, which accomplishes our task. Just ignore.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(),
- collName );
+ // swallow this, it just means this was already updated, which accomplishes our task
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
+ entity.getName(), collName );
}
catch ( Exception ex ) {
logger.error( "Error repersisting entity", ex );
@@ -2778,23 +2779,23 @@ public class CpEntityManager implements EntityManager {
CpWalker walker = new CpWalker( );
- walker.walkCollections( this, application, null, new CpVisitor() {
+ walker.walkCollections( this, application, null, false, new CpVisitor() {
@Override
public void visitCollectionEntry( EntityManager em, String collName, Entity entity ) {
- try {
- em.update( entity );
- po.onProgress( entity );
- }
- catch ( WriteOptimisticVerifyException wo ) {
- //swallow this, it just means this was already updated, which accomplishes our task. Just ignore.
- logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.", entity.getName(),
- collName );
- }
- catch ( Exception ex ) {
- logger.error( "Error repersisting entity", ex );
- }
+ try {
+ em.update( entity );
+ po.onProgress( entity );
+ }
+ catch ( WriteOptimisticVerifyException wo ) {
+ //swallow this, it just means this was already updated, which accomplishes our task.
+ logger.warn( "Someone beat us to updating entity {} in collection {}. Ignoring.",
+ entity.getName(), collName );
+ }
+ catch ( Exception ex ) {
+ logger.error( "Error repersisting entity", ex );
+ }
}
} );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
index 6bd90ec..fab7e16 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpWalker.java
@@ -56,15 +56,17 @@ public class CpWalker {
public void walkCollections(final CpEntityManager em, final EntityRef start,
- String collectionName, final CpVisitor visitor) throws Exception {
+ String collectionName, boolean reverse, final CpVisitor visitor) throws Exception {
- doWalkCollections( em, collectionName, new SimpleId( start.getUuid(), start.getType() ), visitor );
+ doWalkCollections(
+ em, collectionName, reverse, new SimpleId( start.getUuid(), start.getType() ), visitor );
}
private void doWalkCollections(
final CpEntityManager em,
final String collectionName,
+ final boolean reverse,
final Id applicationId,
final CpVisitor visitor ) {
@@ -80,12 +82,20 @@ public class CpWalker {
applicationScope.getApplication().getUuid()
} );
- // only search edge types that start with collections
+ final SearchByEdgeType.Order order;
+ if ( reverse ) {
+ order = SearchByEdgeType.Order.ASCENDING;
+ } else {
+ order = SearchByEdgeType.Order.DESCENDING;
+ }
final String edgeType;
- if ( collectionName != null ) {
+ if ( collectionName == null ) {
+ // only search edge types that end with collections suffix
edgeType = CpNamingUtils.EDGE_COLL_SUFFIX;
+
} else {
+ // only search edges to one collection
edgeType = CpNamingUtils.getEdgeTypeFromCollectionName( collectionName );
}
@@ -99,7 +109,7 @@ public class CpWalker {
logger.debug( "Loading edges of type {} from node {}", edgeType, applicationId );
return gm.loadEdgesFromSource( new SimpleSearchByEdgeType(
- applicationId, edgeType, Long.MAX_VALUE, SearchByEdgeType.Order.DESCENDING, null ) );
+ applicationId, edgeType, Long.MAX_VALUE, order , null ) );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
index b7ca477..4054d77 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManager.java
@@ -704,7 +704,7 @@ public interface EntityManager {
public void flushManagerCaches();
void reindexCollection(
- EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception;
+ EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception;
public void reindex( final EntityManagerFactory.ProgressObserver po ) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
index b7808d8..535a14e 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerFactoryImpl.java
@@ -423,6 +423,11 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
}
@Override
+ public void rebuildCollectionIndex(UUID appId, String collection, boolean reverse, ProgressObserver po) throws Exception {
+ throw new UnsupportedOperationException("Not supported.");
+ }
+
+ @Override
public void rebuildInternalIndexes(ProgressObserver po) throws Exception {
throw new UnsupportedOperationException("Not supported.");
}
@@ -461,12 +466,6 @@ public class EntityManagerFactoryImpl implements EntityManagerFactory, Applicati
throw new UnsupportedOperationException("Not supported in v1");
}
-
- @Override
- public void rebuildCollectionIndex(UUID appId, String collection, ProgressObserver po) {
- throw new UnsupportedOperationException("Not supported.");
- }
-
@Override
public void addIndex(UUID appId, String suffix,final int shards,final int replicas) {
throw new UnsupportedOperationException("Not supported in v1");
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
index d0fe985..be43920 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/EntityManagerImpl.java
@@ -2932,7 +2932,9 @@ public class EntityManagerImpl implements EntityManager {
}
@Override
- public void reindexCollection(EntityManagerFactory.ProgressObserver po, String collectionName) throws Exception {
+ public void reindexCollection(
+ EntityManagerFactory.ProgressObserver po, String collectionName, boolean reverse) throws Exception {
+
throw new UnsupportedOperationException("Not supported.");
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index e853e94..52b6fe4 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -24,6 +24,7 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import org.apache.commons.lang.RandomStringUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -60,7 +61,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
private static final MetricRegistry registry = new MetricRegistry();
private Slf4jReporter reporter;
- private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 3 );
+ private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 10 );
private static final long WRITE_DELAY_MS = 10;
@@ -92,7 +93,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.info("Started rebuildIndex()");
- final EntityManager em = app.getEntityManager();
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
+
+ final EntityManager em = setup.getEmf().getEntityManager( appId );
// ----------------- create a bunch of entities
@@ -101,40 +105,50 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
put("key2", 2000 );
put("key3", "Some value");
}};
- Map<String, Object> cat1map = new HashMap<String, Object>() {{
- put("name", "enzo");
- put("color", "orange");
- }};
- Map<String, Object> cat2map = new HashMap<String, Object>() {{
- put("name", "marquee");
- put("color", "grey");
- }};
- Map<String, Object> cat3map = new HashMap<String, Object>() {{
- put("name", "bertha");
- put("color", "tabby");
- }};
- Entity cat1 = em.create("cat", cat1map );
- Entity cat2 = em.create("cat", cat2map );
- Entity cat3 = em.create("cat", cat3map );
+// Map<String, Object> cat1map = new HashMap<String, Object>() {{
+// put("name", "enzo");
+// put("color", "orange");
+// }};
+// Map<String, Object> cat2map = new HashMap<String, Object>() {{
+// put("name", "marquee");
+// put("color", "grey");
+// }};
+// Map<String, Object> cat3map = new HashMap<String, Object>() {{
+// put("name", "bertha");
+// put("color", "tabby");
+// }};
+//
+// Entity cat1 = em.create("cat", cat1map );
+// Entity cat2 = em.create("cat", cat2map );
+// Entity cat3 = em.create("cat", cat3map );
final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
int entityCount = 0;
+ int herderCount = 0;
+ int shepardCount = 0;
while ( System.currentTimeMillis() < stopTime ) {
final Entity entity;
try {
entityMap.put("key", entityCount );
- entity = em.create("testType", entityMap );
+
+ if ( entityCount % 2 == 0 ) {
+ entity = em.create("catherder", entityMap);
+ herderCount++;
+ } else {
+ entity = em.create("catshepard", entityMap);
+ shepardCount++;
+ }
em.refreshIndex();
- em.createConnection(entity, "herds", cat1);
- em.createConnection(entity, "herds", cat2);
- em.createConnection(entity, "herds", cat3);
+// em.createConnection(entity, "herds", cat1);
+// em.createConnection(entity, "herds", cat2);
+// em.createConnection(entity, "herds", cat3);
} catch (Exception ex) {
throw new RuntimeException("Error creating entity", ex);
@@ -155,24 +169,25 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should work fine
logger.debug("Read the data");
- readData("testTypes", entityCount );
+ readData( em, "catherders", herderCount, 0);
+ readData( em, "catshepards", shepardCount, 0);
// ----------------- delete the system and application indexes
- logger.debug("Deleting app index and system app index");
- deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+ logger.debug("Deleting app index index");
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
deleteIndex( em.getApplicationId() );
// ----------------- test that we can read them, should fail
logger.debug("Reading data, should fail this time ");
try {
- readData( "testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 0 );
fail("should have failed to read data");
} catch (Exception expected) {}
- // ----------------- rebuild index
+ // ----------------- rebuild index for catherders only
logger.debug("Preparing to rebuild all indexes");;
@@ -203,7 +218,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
- setup.getEmf().rebuildApplicationIndexes( em.getApplicationId(), po );
+ setup.getEmf().rebuildCollectionIndex( em.getApplicationId(), "catherders", false, po );
reporter.report();
registry.remove( meterName );
@@ -214,9 +229,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
fail();
}
- // ----------------- test that we can read them
+ // ----------------- test that we can read the catherder collection and not the catshepard
- readData( "testTypes", entityCount );
+ readData( em, "catherders", herderCount, 0 );
+ readData( em, "catshepards", 0, 0 );
}
@@ -225,7 +241,10 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.info("Started rebuildIndex()");
- final EntityManager em = app.getEntityManager();
+ String rand = RandomStringUtils.randomAlphanumeric(5);
+ final UUID appId = setup.createApplication("org_" + rand, "app_" + rand);
+
+ final EntityManager em = setup.getEmf().getEntityManager(appId);
// ----------------- create a bunch of entities
@@ -288,19 +307,19 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them, should work fine
logger.debug("Read the data");
- readData("testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 3 );
// ----------------- delete the system and application indexes
logger.debug("Deleting app index and system app index");
- deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
+ //deleteIndex( CpNamingUtils.SYSTEM_APP_ID );
deleteIndex( em.getApplicationId() );
// ----------------- test that we can read them, should fail
logger.debug("Reading data, should fail this time ");
try {
- readData( "testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 3 );
fail("should have failed to read data");
} catch (Exception expected) {}
@@ -349,7 +368,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
// ----------------- test that we can read them
- readData( "testTypes", entityCount );
+ readData( em, "testTypes", entityCount, 3 );
}
/**
@@ -369,9 +388,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
- private int readData( String collectionName, int expected ) throws Exception {
+ private int readData( EntityManager em,
+ String collectionName, int expectedEntities, int expectedConnections ) throws Exception {
- EntityManager em = app.getEntityManager();
em.refreshIndex();
Query q = Query.fromQL("select * where key1=1000");
@@ -387,7 +406,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
Results catResults = em.searchConnectedEntities(e,
Query.fromQL("select *").setConnectionType( "herds" ));
- assertEquals( 3, catResults.size() );
+ assertEquals( expectedConnections, catResults.size() );
if ( count % 100 == 0 ) {
logger.info( "read {} entities", count);
@@ -405,9 +424,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
}
- if ( expected != -1 && expected != count ) {
+ if ( expectedEntities != -1 && expectedEntities != count ) {
throw new RuntimeException("Did not get expected "
- + expected + " entities, instead got " + count );
+ + expectedEntities + " entities, instead got " + count );
}
return count;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/2f077f1b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
----------------------------------------------------------------------
diff --git a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
index e566512..7fcfc57 100644
--- a/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
+++ b/stack/services/src/test/java/org/apache/usergrid/management/importer/ImportCollectionIT.java
@@ -110,7 +110,8 @@ public class ImportCollectionIT {
bucketPrefix = System.getProperty( "bucketName" );
// start the scheduler after we're all set up
- JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance().getSpringResource().getBean( JobSchedulerService.class );
+ JobSchedulerService jobScheduler = ConcurrentProcessSingleton.getInstance()
+ .getSpringResource().getBean( JobSchedulerService.class );
if ( jobScheduler.state() != Service.State.RUNNING ) {
jobScheduler.startAsync();
jobScheduler.awaitRunning();
@@ -481,7 +482,8 @@ public class ImportCollectionIT {
ImportService importService = setup.getImportService();
- Import importEntity = importService.schedule(em.getApplication().getUuid(), new HashMap<String, Object>() {{
+ Import importEntity = importService.schedule(em.getApplication().getUuid(),
+ new HashMap<String, Object>() {{
put( "path", organization.getName() + em.getApplication().getName() );
put( "organizationId", organization.getUuid() );
put( "applicationId", em.getApplication().getUuid() );