You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/23 19:30:54 UTC
[44/50] [abbrv] incubator-usergrid git commit: move initialize index
move initialize index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/f69bcebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/f69bcebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/f69bcebe
Branch: refs/heads/USERGRID-347
Commit: f69bcebe4439a408a4d2baa87a3322142c506f93
Parents: 2af7489
Author: Shawn Feldman <sf...@apache.org>
Authored: Fri Mar 20 17:22:22 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Fri Mar 20 17:22:22 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 13 +++++++---
.../usergrid/corepersistence/CpSetup.java | 13 +++++++++-
.../index/ApplicationEntityIndex.java | 5 ++++
.../usergrid/persistence/index/EntityIndex.java | 6 +----
.../impl/EsApplicationEntityIndexImpl.java | 12 +++++++++
.../index/impl/EsEntityIndexImpl.java | 9 -------
.../persistence/index/impl/EntityIndexTest.java | 26 +++++++++++---------
.../index/impl/IndexLoadTestsIT.java | 4 +--
8 files changed, 56 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index 1388462..95ffbaa 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -25,7 +25,9 @@ import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.atomic.AtomicBoolean;
+import com.hazelcast.core.IdGenerator;
import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeansException;
@@ -92,6 +94,7 @@ import static org.apache.usergrid.persistence.Schema.TYPE_APPLICATION;
public class CpEntityManagerFactory implements EntityManagerFactory, ApplicationContextAware {
private static final Logger logger = LoggerFactory.getLogger( CpEntityManagerFactory.class );
+ private final EntityIndexFactory entityIndexFactory;
private ApplicationContext applicationContext;
@@ -129,6 +132,7 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
this.counterUtils = counterUtils;
this.injector = injector;
this.entityIndex = injector.getInstance(EntityIndex.class);
+ this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
this.managerCache = injector.getInstance( ManagerCache.class );
this.metricsFactory = injector.getInstance( MetricsFactory.class );
@@ -160,7 +164,9 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
em.getApplication();
}
- entityIndex.initializeIndex();
+ ApplicationScope appScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ) );
+ ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(appScope);
+ applicationEntityIndex.initializeIndex();
entityIndex.refresh();
@@ -722,9 +728,10 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
public void rebuildApplicationIndexes( UUID appId, ProgressObserver po ) throws Exception {
EntityManager em = getEntityManager( appId );
-
+ ApplicationScope applicationScope = new ApplicationScopeImpl( new SimpleId( CpNamingUtils.SYSTEM_APP_ID, "application" ));
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- entityIndex.initializeIndex();
+ ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
+ applicationEntityIndex.initializeIndex();
em.reindex(po);
em.reindex( po );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
index a9e6cee..1d75e47 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpSetup.java
@@ -20,7 +20,13 @@ package org.apache.usergrid.corepersistence;
import java.util.UUID;
import com.google.inject.Binding;
+import org.apache.usergrid.corepersistence.util.CpNamingUtils;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
import org.apache.usergrid.persistence.index.EntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -67,6 +73,7 @@ public class CpSetup implements Setup {
private final EntityManagerFactory emf;
private final EntityIndex entityIndex;
+ private final EntityIndexFactory entityIndexFactory;
/**
@@ -79,6 +86,8 @@ public class CpSetup implements Setup {
this.cass = cassandraService;
this.injector = injector;
this.entityIndex = injector.getInstance(EntityIndex.class);
+ this.entityIndexFactory = injector.getInstance(EntityIndexFactory.class);
+
}
@@ -98,8 +107,10 @@ public class CpSetup implements Setup {
setupStaticKeyspace();
+ ApplicationScope applicationScope = new ApplicationScopeImpl(new SimpleId( CpNamingUtils.SYSTEM_APP_ID,"application"));
+ ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(applicationScope);
//force the EMF creation of indexes before creating the default applications
- entityIndex.initializeIndex();
+ applicationEntityIndex.initializeIndex();
injector.getInstance( DataMigrationManager.class ).migrate();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index 34967bd..4808d8a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -32,6 +32,11 @@ public interface ApplicationEntityIndex {
/**
+ *
+ */
+ public void initializeIndex();
+
+ /**
* Create the index batch.
*/
public EntityIndexBatch createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 854c976..af6b013 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -38,11 +38,7 @@ import java.util.concurrent.Future;
*/
public interface EntityIndex extends CPManager {
- /**
- * This should ONLY ever be called once on application create.
- * Otherwise we're introducing slowness into our system
- */
- public void initializeIndex();
+
/**
* Create an index and add to alias, will create alias and remove any old index from write alias if alias already exists
* @param indexSuffix index name
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index 56f8030..2d01697 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -126,6 +126,18 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
+ public void initializeIndex() {
+ final int numberOfShards = indexFig.getNumberOfShards();
+ final int numberOfReplicas = indexFig.getNumberOfReplicas();
+ indexCache.invalidate(alias);
+ String[] indexes = entityIndex.getUniqueIndexes();
+ if(indexes == null || indexes.length==0) {
+ entityIndex.addIndex(null, numberOfShards, numberOfReplicas, indexFig.getWriteConsistencyLevel());
+ }
+
+ }
+
+ @Override
public EntityIndexBatch createBatch() {
EntityIndexBatch batch = new EsEntityIndexBatchImpl(
applicationScope, indexBatchBufferProducer, entityIndex, indexIdentifier );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 4290f23..259fa55 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -139,16 +139,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
- @Override
- public void initializeIndex() {
- final int numberOfShards = config.getNumberOfShards();
- final int numberOfReplicas = config.getNumberOfReplicas();
- String[] indexes = getIndexesFromEs(AliasType.Write);
- if(indexes == null || indexes.length==0) {
- addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
- }
- }
@Override
public void addIndex(final String indexSuffix,final int numberOfShards, final int numberOfReplicas, final String writeConsistency) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index b562ccf..85570f1 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -91,10 +91,10 @@ public class EntityIndexTest extends BaseIT {
@Test
public void testIndex() throws IOException, InterruptedException {
Id appId = new SimpleId("application");
- ei.initializeIndex();
ApplicationScope applicationScope = new ApplicationScopeImpl(appId);
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
+ entityIndex.initializeIndex();
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl(appId, "things");
@@ -158,7 +158,7 @@ public class EntityIndexTest extends BaseIT {
final ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
final IndexScope indexScope = new IndexScopeImpl(appId, "things");
final String entityType = "thing";
- ei.initializeIndex();
+ entityIndex.initializeIndex();
final CountDownLatch latch = new CountDownLatch(threads);
final AtomicLong failTime=new AtomicLong(0);
InputStream is = this.getClass().getResourceAsStream( "/sample-large.json" );
@@ -201,9 +201,9 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- ei.initializeIndex();
+ entityIndex.initializeIndex();
for(int i=0;i<10;i++) {
- ei.initializeIndex();
+ entityIndex.initializeIndex();
}
}
@@ -215,7 +215,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- ei.initializeIndex();
+ entityIndex.initializeIndex();
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
@@ -328,7 +328,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- ei.initializeIndex();
+ entityIndex.initializeIndex();
Map entityMap = new HashMap() {{
put( "name", "Ferrari 212 Inter" );
@@ -459,7 +459,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- ei.initializeIndex();
+ entityIndex.initializeIndex();
final String middleName = "middleName" + UUIDUtils.newTimeUUID();
@@ -508,7 +508,7 @@ public class EntityIndexTest extends BaseIT {
IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- ei.initializeIndex();
+ entityIndex.initializeIndex();
entityIndex.createBatch();
// Bill has favorites as string, age as string and retirement goal as number
@@ -576,11 +576,13 @@ public class EntityIndexTest extends BaseIT {
@Test
public void healthTest() {
-
+ Id appId = new SimpleId( "entityindextest" );
+ Id ownerId = new SimpleId( "multivaluedtype" );
+ ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
assertNotEquals( "cluster should be ok", Health.RED, ei.getClusterHealth() );
assertEquals( "index should be ready", Health.GREEN, ei.getIndexHealth() );
-
- ei.initializeIndex();
+ ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
+ entityIndex.initializeIndex();
ei.refresh();
assertNotEquals( "cluster should be fine", Health.RED, ei.getIndexHealth() );
@@ -600,7 +602,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationEntityIndex entityIndex = eif.createApplicationEntityIndex(applicationScope);
- ei.initializeIndex();
+ entityIndex.initializeIndex();
final EntityIndexBatch batch = entityIndex.createBatch();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/f69bcebe/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
index 623f9dd..44a9743 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/IndexLoadTestsIT.java
@@ -81,12 +81,12 @@ public class IndexLoadTestsIT extends BaseIT {
final Id applicationId = new SimpleId( applicationUUID, "application" );
final ApplicationScope scope = new ApplicationScopeImpl( applicationId );
+ ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(scope);
//create our index if it doesn't exist
- index.initializeIndex();
+ applicationEntityIndex.initializeIndex();
- ApplicationEntityIndex applicationEntityIndex = entityIndexFactory.createApplicationEntityIndex(scope);
final Observable<Entity> createEntities = createStreamFromWorkers( applicationEntityIndex, applicationId );
//run them all