You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/31 23:01:23 UTC
[45/50] incubator-usergrid git commit: refactor initiaization index
logic
refactor initiaization index logic
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/07cb9dfd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/07cb9dfd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/07cb9dfd
Branch: refs/heads/USERGRID-473
Commit: 07cb9dfd6563d974d8c6b9764633be44e389a4d0
Parents: 3a31836
Author: Shawn Feldman <sf...@apache.org>
Authored: Mon Mar 30 11:45:15 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Mon Mar 30 11:45:15 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 14 +--------
.../index/ApplicationEntityIndex.java | 5 ----
.../usergrid/persistence/index/EntityIndex.java | 10 +++----
.../impl/EsApplicationEntityIndexImpl.java | 12 --------
.../index/impl/EsEntityIndexImpl.java | 13 +++++++++
.../migration/EsIndexDataMigrationImpl.java | 4 ++-
.../persistence/index/impl/EntityIndexTest.java | 30 +++++++++++---------
.../index/impl/IndexLoadTestsIT.java | 4 ---
8 files changed, 38 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 3cfc1a4..99204a8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -164,9 +164,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.getApplication();
}
- ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) );
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope);
- applicationEntityIndex.initializeIndex();
+ entityIndex.initialize();
entityIndex.refresh();
@@ -265,10 +263,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
throw new ApplicationAlreadyExistsException( appName );
}
- ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( applicationId,"application"));
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
-
getSetup().setupApplicationKeyspace( applicationId, appName );
final Optional<UUID> cachedValue = orgApplicationCache.getOrganizationId( organizationName );
@@ -730,12 +724,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
EntityManager em = getEntityManager( appId );
- ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ));
- //explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
- applicationEntityIndex.initializeIndex();
- em.reindex(po);
-
em.reindex( po );
logger.info("\n\nRebuilt index for applicationId {} \n", appId );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index cb1b6f0..86e97c5 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -30,11 +30,6 @@ public interface ApplicationEntityIndex {
/**
- *
- */
- public void initializeIndex();
-
- /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 6783fb0..ff9008e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -48,28 +48,28 @@ public interface EntityIndex extends CPManager {
* @param replicas
* @param writeConsistency
*/
- public void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
+ void addIndex(final String indexSuffix, final int shards, final int replicas, final String writeConsistency);
/**
* Refresh the index.
*/
- public void refresh();
+ void refresh();
/**
* Check health of cluster.
*/
- public Health getClusterHealth();
+ Health getClusterHealth();
/**
* Check health of this specific index.
*/
- public Health getIndexHealth();
-
+ Health getIndexHealth();
+ void initialize();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 9248980..84845d0 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -116,18 +116,6 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public void initializeIndex() {
- final int numberOfShards = indexFig.getNumberOfShards();
- final int numberOfReplicas = indexFig.getNumberOfReplicas();
- indexCache.invalidate(alias);
- String[] indexes = entityIndex.getUniqueIndexes();
- if(indexes == null || indexes.length==0) {
- entityIndex.addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
- }
-
- }
-
- @Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 055af2a..ee28939 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -131,6 +131,19 @@ public class EsEntityIndexImpl implements AliasedEntityIndex,VersionedData {
}
@Override
+ public void initialize(){
+ final int numberOfShards = indexFig.getNumberOfShards();
+ final int numberOfReplicas = indexFig.getNumberOfReplicas();
+ aliasCache.invalidate(alias);
+ String[] reads = getIndexes(AliasedEntityIndex.AliasType.Read);
+ String[] writes = getIndexes(AliasedEntityIndex.AliasType.Write);
+
+ if(reads.length==0 || writes.length==0) {
+ addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
+ }
+ }
+
+ @Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
try {
//get index name with suffix attached
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
index 123527a..3706722 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/migration/EsIndexDataMigrationImpl.java
@@ -62,6 +62,8 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
final AdminClient adminClient = provider.getClient().admin();
final int latestVersion = dataVersion.getImplementationVersion();
+ entityIndex.initialize();
+
observer.start();
try {
migrationDataProvider.getData().flatMap(applicationScope -> {
@@ -74,7 +76,7 @@ public class EsIndexDataMigrationImpl implements DataMigration<ApplicationScope>
aliasesRequestBuilder = adminClient.indices().prepareAliases();
// add read alias
try {
- aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias());
+ aliasesRequestBuilder.addAlias(index, indexIdentifier.getAlias().getReadAlias()).get();
} catch (InvalidAliasNameException e) {
log.debug("Failed to add alias due to name conflict",e);
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 2c03635..27f0b9d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -30,9 +30,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.usergrid.persistence.index.*;
-import org.junit.Ignore;
-import org.junit.Rule;
-import org.junit.Test;
+import org.junit.*;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -87,6 +85,10 @@ public class EntityIndexTest extends BaseIT {
@Rule
public MigrationManagerRule migrationManagerRule;
+ @Before
+ public void setup(){
+ ei.initialize();
+ }
@Test
public void testIndex() throws IOException, InterruptedException {
@@ -94,7 +96,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl(appId);
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl(appId, "things");
@@ -158,7 +160,7 @@ public class EntityIndexTest extends BaseIT {
final ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
final IndexScope indexScope = new IndexScopeImpl(appId, "things");
final String entityType = "thing";
- entityIndex.initializeIndex();
+
final CountDownLatch latch = new CountDownLatch(threads);
final AtomicLong failTime=new AtomicLong(0);
InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
@@ -201,9 +203,9 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
for(int i=0;i<10;i++) {
- entityIndex.initializeIndex();
+
}
}
@@ -215,7 +217,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
@@ -328,7 +330,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
Map entityMap = new HashMap() {{
put( "name", "Ferrari 212 Inter" );
@@ -460,7 +462,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
@@ -509,7 +511,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
entityIndex.createBatch();
// Bill has favorites as string, age as string and retirement goal as number
@@ -581,9 +583,9 @@ public class EntityIndexTest extends BaseIT {
Id ownerId = new SimpleId( "multivaluedtype" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
- assertEquals( "index should be ready", Health.GREEN, ei.getIndexHealth() );
+ assertEquals("index should be ready", Health.GREEN, ei.getIndexHealth());
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
ei.refresh();
assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
@@ -603,7 +605,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- entityIndex.initializeIndex();
+
final EntityIndexBatch batch = entityIndex.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/07cb9dfd/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index 0275e53..b03f09d 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -135,10 +135,6 @@ public class IndexLoadTestsIT extends BaseIT {
final IndexScope indexScope = new IndexScopeImpl( applicationId, "test" );
- final ApplicationEntityIndex appEntityIndex = entityIndexFactory.createApplicationEntityIndex( scope );
-
- appEntityIndex.initializeIndex();
-
batchWriteTPS = metricsFactory.getMeter( IndexLoadTestsIT.class, "write.tps" );
batchWriteTimer = metricsFactory.getTimer( IndexLoadTestsIT.class, "write.timer" );