You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/05/26 20:13:01 UTC
[02/22] incubator-usergrid git commit: Update of Apis and tests
Update of Apis and tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/dbfff997
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/dbfff997
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/dbfff997
Branch: refs/heads/USERGRID-608
Commit: dbfff9970f844f2264113fc05ee290aa3ca0ae50
Parents: 30e0e45
Author: Todd Nine <tn...@apigee.com>
Authored: Mon May 11 09:27:28 2015 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Tue May 12 10:29:41 2015 -0700
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 14 ------
.../corepersistence/index/ReIndexAction.java | 4 ++
.../corepersistence/index/ReIndexService.java | 11 ++--
.../index/ReIndexServiceImpl.java | 5 +-
.../persistence/EntityManagerFactory.java | 2 -
.../PerformanceEntityRebuildIndexTest.java | 53 ++++++--------------
6 files changed, 27 insertions(+), 62 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index e796545..13b6433 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -687,20 +687,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
}
}
- @Override
- public ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection ) {
- throw new UnsupportedOperationException( "Implement me" );
-//
-// EntityManager em = getEntityManager( appId );
-//
-// //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
-// Application app = em.getApplication();
-//
-// em.reindexCollection(po, collectionName, reverse);
-//
-// logger.info("\n\nRebuilt index for application {} id {} collection {}\n",
-// new Object[]{app.getName(), appId, collectionName});
- }
@Override
public void addIndex(final String indexSuffix,final int shards,final int replicas, final String writeConsistency){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
index 086b2aa..b878246 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexAction.java
@@ -29,5 +29,9 @@ import org.apache.usergrid.persistence.collection.serialization.impl.migration.E
@FunctionalInterface
public interface ReIndexAction {
+ /**
+ * Index this entity with the specified scope
+ * @param entityIdScope
+ */
void index( final EntityIdScope entityIdScope );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
index 91409fe..e594ad3 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexService.java
@@ -44,12 +44,15 @@ public interface ReIndexService {
/**
* Perform an index rebuild
- * @param appId
- * @param collection
+ *
+ * @param appId The applicationId to re-index, or all applications if absent
+ * @param collection The collection name to re-index. Otherwise all collections in an app will be used.
+ * @param cursor An optional cursor to resume processing
+ * @param startTimestamp The time to start indexing from. All edges >= this time will be indexed.
* @return
*/
- IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> collectionName, final Optional<String> cursor,
- final Optional<Long> startTimestamp );
+ IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor,
+ final Optional<Long> startTimestamp);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
index be5bcab..bd1bff9 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/index/ReIndexServiceImpl.java
@@ -84,8 +84,7 @@ public class ReIndexServiceImpl implements ReIndexService {
@Override
- public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection,
- final Optional<String> collectionName, final Optional<String> cursor,
+ public IndexResponse rebuildIndex( final Optional<UUID> appId, final Optional<String> collection, final Optional<String> cursor,
final Optional<Long> startTimestamp ) {
//load our last emitted Scope if a cursor is present
@@ -100,7 +99,7 @@ public class ReIndexServiceImpl implements ReIndexService {
//create an observable that loads each entity and indexes it, start it running with publish
final ConnectableObservable<EdgeScope> runningReIndex =
- allEntityIdsObservable.getEdgesToEntities( applicationScopes, collectionName, startTimestamp )
+ allEntityIdsObservable.getEdgesToEntities( applicationScopes, collection, startTimestamp )
//for each edge, create our scope and index on it
.doOnNext( edge -> indexService.index( new EntityIdScope( edge.getApplicationScope(), edge.getEdge().getTargetNode() ) ) ).publish();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
index b3f4b62..cd7e515 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/EntityManagerFactory.java
@@ -174,8 +174,6 @@ public interface EntityManagerFactory {
/** For testing purposes */
public void flushEntityManagerCaches();
- ReIndexService.IndexResponse rebuildCollectionIndex( Optional<UUID> appId, Optional<String> collection );
-
/**
* Add a new index to the application for scale
* @param suffix unique indentifier for additional index
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/dbfff997/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
index df2a762..a17c925 100644
--- a/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/persistence/PerformanceEntityRebuildIndexTest.java
@@ -26,6 +26,8 @@ import java.util.concurrent.TimeUnit;
import com.google.common.base.Optional;
import org.apache.commons.lang.RandomStringUtils;
+
+import org.apache.usergrid.corepersistence.index.ReIndexService;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.junit.After;
import org.junit.Before;
@@ -63,9 +65,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
private static final MetricRegistry registry = new MetricRegistry();
private Slf4jReporter reporter;
- private static final long RUNTIME_MS = TimeUnit.SECONDS.toMillis( 10 );
+ private static final int ENTITIES_TO_INDEX = 2000;
- private static final long WRITE_DELAY_MS = 10;
@@ -99,6 +100,8 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
final EntityManager em = setup.getEmf().getEntityManager( appId );
+ final ReIndexService reIndexService = setup.getInjector().getInstance( ReIndexService.class );
+
// ----------------- create a bunch of entities
Map<String, Object> entityMap = new HashMap<String, Object>() {{
@@ -107,37 +110,18 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
put("key3", "Some value");
}};
-// Map<String, Object> cat1map = new HashMap<String, Object>() {{
-// put("name", "enzo");
-// put("color", "orange");
-// }};
-// Map<String, Object> cat2map = new HashMap<String, Object>() {{
-// put("name", "marquee");
-// put("color", "grey");
-// }};
-// Map<String, Object> cat3map = new HashMap<String, Object>() {{
-// put("name", "bertha");
-// put("color", "tabby");
-// }};
-//
-// Entity cat1 = em.create("cat", cat1map );
-// Entity cat2 = em.create("cat", cat2map );
-// Entity cat3 = em.create("cat", cat3map );
-
- final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
- int entityCount = 0;
int herderCount = 0;
int shepardCount = 0;
- while ( System.currentTimeMillis() < stopTime ) {
+ for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
final Entity entity;
try {
- entityMap.put("key", entityCount );
+ entityMap.put("key", i );
- if ( entityCount % 2 == 0 ) {
+ if ( i % 2 == 0 ) {
entity = em.create("catherder", entityMap);
herderCount++;
} else {
@@ -156,15 +140,13 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
}
entityRefs.add(new SimpleEntityRef( entity.getType(), entity.getUuid() ) );
- if ( entityCount % 10 == 0 ) {
- logger.info("Created {} entities", entityCount );
+ if ( i % 10 == 0 ) {
+ logger.info("Created {} entities", i );
}
- entityCount++;
- try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {}
}
- logger.info("Created {} entities", entityCount);
+ logger.info("Created {} entities", ENTITIES_TO_INDEX);
app.refreshIndex();
// ----------------- test that we can read them, should work fine
@@ -182,7 +164,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.debug("Reading data, should fail this time ");
try {
- readData( em, "testTypes", entityCount, 0 );
+ readData( em, "testTypes", ENTITIES_TO_INDEX, 0 );
fail("should have failed to read data");
} catch (Exception expected) {}
@@ -214,10 +196,7 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
try {
-// // do it forwards
- setup.getEmf().rebuildCollectionIndex(Optional.of(em.getApplicationId()), Optional.<String>of("catherders"));
-//
-// // and backwards, just to make sure both cases are covered
+ reIndexService.rebuildIndex( Optional.of( em.getApplicationId()), Optional.<String>of("catherders"), Optional.absent(), Optional.absent() );
reporter.report();
registry.remove( meterName );
@@ -269,11 +248,9 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
Entity cat2 = em.create("cat", cat2map );
Entity cat3 = em.create("cat", cat3map );
- final long stopTime = System.currentTimeMillis() + RUNTIME_MS;
-
List<EntityRef> entityRefs = new ArrayList<EntityRef>();
int entityCount = 0;
- while ( System.currentTimeMillis() < stopTime ) {
+ for (int i = 0; i < ENTITIES_TO_INDEX; i++) {
final Entity entity;
@@ -295,8 +272,6 @@ public class PerformanceEntityRebuildIndexTest extends AbstractCoreIT {
logger.info("Created {} entities", entityCount );
}
- entityCount++;
- try { Thread.sleep( WRITE_DELAY_MS ); } catch (InterruptedException ignored ) {}
}
logger.info("Created {} entities", entityCount);