You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/20 22:22:35 UTC
[03/29] incubator-usergrid git commit: single index
single index
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/346eba98
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/346eba98
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/346eba98
Branch: refs/heads/USERGRID-405
Commit: 346eba98ed1080940730fb617951e6172a1683bb
Parents: 1199e47
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 18 14:56:36 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 18 14:56:36 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManagerFactory.java | 3 -
.../usergrid/persistence/index/EntityIndex.java | 21 ------
.../persistence/index/IndexIdentifier.java | 16 +----
.../usergrid/persistence/index/SearchTypes.java | 14 +++-
.../index/impl/EsEntityIndexBatchImpl.java | 6 +-
.../index/impl/EsEntityIndexImpl.java | 69 +++++---------------
.../persistence/index/impl/IndexingUtils.java | 8 ++-
.../persistence/index/impl/EntityIndexTest.java | 65 +-----------------
8 files changed, 42 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
index fe4d828..2bdb54c 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManagerFactory.java
@@ -292,7 +292,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityManager appEm = getEntityManager( applicationId );
appEm.create( applicationId, TYPE_APPLICATION, properties );
- appEm.createIndex();
appEm.resetRoles();
appEm.refreshIndex();
@@ -706,7 +705,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityManager em = getEntityManager( appId );
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- em.createIndex();
Application app = em.getApplication();
em.reindex( po );
@@ -760,7 +758,6 @@ public class CpEntityManagerFactory implements EntityManagerFactory, Application
EntityManager em = getEntityManager( appId );
//explicitly invoke create index, we don't know if it exists or not in ES during a rebuild.
- em.createIndex();
Application app = em.getApplication();
em.reindexCollection( po, collectionName, reverse );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index aa2bf8e..c8840fa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -68,27 +68,6 @@ public interface EntityIndex {
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query );
/**
- * Get the candidate results of all versions of the entity for this id.
- * @param indexScope The scope of the index to search in
- * @param id The id to search within.
- */
- public CandidateResults getEntityVersions(final IndexScope indexScope, final Id id);
-
-// /**
-// * Create a delete method that deletes by Id. This will delete all documents from ES with the same entity Id,
-// * effectively removing all versions of an entity from all index scopes
-// * @param entityId The entityId to remove
-// */
-// public Future deleteAllVersionsOfEntity(final Id entityId );
-//
-// /**
-// * Takes all the previous versions of the current entity and deletes all previous versions
-// * @param id The id to remove
-// * @param version The max version to retain
-// */
-// public Future deletePreviousVersions(final Id id, final UUID version);
-
- /**
* Refresh the index.
*/
public void refresh();
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
index f23dc36..8465d84 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexIdentifier.java
@@ -40,7 +40,7 @@ public class IndexIdentifier{
* @return
*/
public IndexAlias getAlias() {
- return new IndexAlias(config,getIndexBase());
+ return new IndexAlias(config,config.getIndexPrefix());
}
/**
@@ -50,22 +50,12 @@ public class IndexIdentifier{
*/
public String getIndex(String suffix) {
if (suffix != null) {
- return getIndexBase() + "_" + suffix;
+ return config.getIndexPrefix() + "_" + suffix;
} else {
- return getIndexBase();
+ return config.getIndexPrefix();
}
}
- /**
- * returns the base name for index which will be used to add an alias and index
- * @return
- */
- private String getIndexBase() {
- StringBuilder sb = new StringBuilder();
- sb.append(config.getIndexPrefix()).append(IndexingUtils.SEPARATOR);
- IndexingUtils.idString(sb, applicationScope.getApplication());
- return sb.toString();
- }
public class IndexAlias{
private final String readAlias;
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
index 35b3a8b..1a97da2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchTypes.java
@@ -20,6 +20,10 @@ package org.apache.usergrid.persistence.index;/*
*/
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.index.impl.IndexingUtils;
+import org.apache.usergrid.persistence.model.entity.Id;
+
import java.util.Arrays;
@@ -37,11 +41,17 @@ public class SearchTypes {
private SearchTypes( final String... types ) {this.types = types;}
- public String[] getTypeNames() {
- return types;
+ public String[] getTypeNames(ApplicationScope applicationScope) {
+ String[] typeNames = new String[types.length];
+ int i =0 ;
+ for(String type : types){
+ typeNames[i++] = IndexingUtils.getType(applicationScope,type);
+ }
+ return typeNames;
}
+
/**
* Create a search that will search on the specified types
* @param types
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index 23b9041..977fe51 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -117,9 +117,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
String indexId = createIndexDocId( entity, context );
log.debug( "Indexing entity documentId {} data {} ", indexId, entityAsMap );
- final String entityType = entity.getId().getType();
-
-
+ final String entityType =IndexingUtils.getType(applicationScope, entity.getId());
container.addIndexRequest(new IndexRequest(alias.getWriteAlias(), entityType, indexId, entityAsMap));
return this;
@@ -134,7 +132,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
ValidationUtils.verifyVersion( version );
final String context = createContextName(indexScope);
- final String entityType = id.getType();
+ final String entityType =IndexingUtils.getType(applicationScope, id);
final String indexId = createIndexDocId( id, version, context );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 598c8d9..110b320 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -199,9 +199,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final int numberOfShards = config.getNumberOfShards();
final int numberOfReplicas = config.getNumberOfReplicas();
String[] indexes = getIndexesFromEs(AliasType.Write);
- if(indexes == null || indexes.length==0) {
+// if(indexes == null || indexes.length==0) {
addIndex(null, numberOfShards, numberOfReplicas, config.getWriteConsistencyLevel());
- }
+// }
}
@Override
@@ -233,21 +233,21 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
//ONLY add the alias if we create the index, otherwise we're going to overwrite production settings
- /**
- * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!!
- */
-
- //We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that
- //may have been set via other administrative endpoint
-
- addAlias(normalizedSuffix);
-
- testNewIndex();
logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
} catch (IndexAlreadyExistsException e) {
logger.info("Index Name [{}] already exists", indexName);
}
+ /**
+ * DO NOT MOVE THIS LINE OF CODE UNLESS YOU REALLY KNOW WHAT YOU'RE DOING!!!!
+ */
+
+ //We do NOT want to create an alias if the index already exists, we'll overwrite the indexes that
+ //may have been set via other administrative endpoint
+
+ addAlias(normalizedSuffix);
+
+ testNewIndex();
@@ -347,7 +347,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
// to receive documents. Occasionally we see errors.
// See this post: http://s.apache.org/index-missing-exception
- logger.debug( "Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
+ logger.debug("Testing new index name: read {} write {}", alias.getReadAlias(), alias.getWriteAlias());
final RetryOperation retryOperation = new RetryOperation() {
@Override
@@ -410,12 +410,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
- final String context = IndexingUtils.createContextName( indexScope );
- final String[] entityTypes = searchTypes.getTypeNames();
-
+ final String context = IndexingUtils.createContextName(indexScope);
+ final String[] entityTypes = searchTypes.getTypeNames(applicationScope);
QueryBuilder qb = query.createQueryBuilder(context);
-
-
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
@@ -635,42 +632,6 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
- @Override
- public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
-
- //since we don't have paging inputs, there's no point in executing a query for paging.
-
- final String context = IndexingUtils.createContextName(scope);
- final SearchTypes searchTypes = SearchTypes.fromTypes(id.getType());
-
- final QueryBuilder queryBuilder =
- QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context );
-
- final SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames())
- .setScroll(cursorTimeout + "m")
- .setQuery(queryBuilder);
-
- final SearchResponse searchResponse;
- try {
- //Added For Graphite Metrics
- Timer.Context timeEntityIndex = getVersionsTimer.time();
- searchResponse = srb.execute().actionGet();
- timeEntityIndex.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch" );
- failureMonitor.fail( "Unable to execute batch", t);
- throw t;
- }
-
-
- failureMonitor.success();
-
- return parseResults(searchResponse, new Query());
- }
-
-
/**
* Completely delete an index.
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 0df3a2c..aa6d0f3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -230,7 +230,7 @@ public class IndexingUtils {
.startObject("template__long")
.field("match", LONG_PREFIX + "*")
.field("match_mapping_type", "long")
- .startObject("mapping").field("type", "long").field("index","not_analyzed").field(DOC_VALUES_KEY, true).endObject()
+ .startObject("mapping").field("type", "long").field("index", "not_analyzed").field(DOC_VALUES_KEY, true).endObject()
.endObject()
.endObject()
@@ -249,5 +249,11 @@ public class IndexingUtils {
}
+ public static String getType(ApplicationScope applicationScope, Id entityId) {
+ return getType(applicationScope,entityId.getType());
+ }
+ public static String getType(ApplicationScope applicationScope, String type) {
+ return idString(applicationScope.getApplication()) + SEPARATOR + type;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/346eba98/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index ca9bf79..22ca51f 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -271,7 +271,7 @@ public class EntityIndexTest extends BaseIT {
ObjectMapper mapper = new ObjectMapper();
List<Object> sampleJson = mapper.readValue( is, new TypeReference<List<Object>>() {} );
EntityIndexBatch batch = entityIndex.createBatch();
- insertJsonBlob(sampleJson,batch, entityType, indexScope, max,startIndex);
+ insertJsonBlob(sampleJson,batch, entityType, indexScope, max, startIndex);
batch.execute().get();
entityIndex.refresh();
}
@@ -297,10 +297,9 @@ public class EntityIndexTest extends BaseIT {
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
entity.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
batch.index(indexScope, entity);
+ batch.execute().get();
+
- if(count %1000 == 0){
- batch.execute().get();
- }
if ( ++count > max ) {
break;
}
@@ -370,7 +369,6 @@ public class EntityIndexTest extends BaseIT {
private void testQueries(final IndexScope scope, SearchTypes searchTypes, final EntityIndex entityIndex ) {
-
testQuery(scope, searchTypes, entityIndex, "name = 'Morgan Pierce'", 1 );
testQuery(scope, searchTypes, entityIndex, "name = 'morgan pierce'", 1 );
@@ -443,63 +441,6 @@ public class EntityIndexTest extends BaseIT {
}
}
-
- @Test
- public void getEntityVersions() throws Exception {
-
- Id appId = new SimpleId( "application" );
- Id ownerId = new SimpleId( "owner" );
-
- ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
-
- IndexScope indexScope = new IndexScopeImpl( ownerId, "users" );
-
-
-
- EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
- entityIndex.initializeIndex();
-
- final String middleName = "middleName" + UUIDUtils.newTimeUUID();
- Map<String, Object> properties = new LinkedHashMap<String, Object>();
- properties.put( "username", "edanuff" );
- properties.put( "email", "ed@anuff.com" );
- properties.put( "middlename", middleName );
-
- Map entityMap = new HashMap() {{
- put( "username", "edanuff" );
- put( "email", "ed@anuff.com" );
- put( "middlename", middleName );
- }};
-
- final Id userId = new SimpleId("user");
-
- Entity user = EntityIndexMapUtils.fromMap( entityMap );
- EntityUtils.setId( user, userId);
- EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() );
-
-
- final EntityIndexBatch batch = entityIndex.createBatch();
- user.setField(new UUIDField(IndexingUtils.ENTITYID_ID_FIELDNAME, UUID.randomUUID()));
-
- batch.index( indexScope, user );
-
- user.setField( new StringField( "address1", "1782 address st" ) );
- batch.index( indexScope, user );
- user.setField( new StringField( "address2", "apt 508" ) );
- batch.index( indexScope, user );
- user.setField( new StringField( "address3", "apt 508" ) );
- batch.index( indexScope, user);
- batch.execute().get();
- entityIndex.refresh();
-
- CandidateResults results = entityIndex.getEntityVersions(indexScope, user.getId() );
-
- assertEquals(1, results.size());
- assertEquals( results.get( 0 ).getId(), user.getId() );
- assertEquals( results.get(0).getVersion(), user.getVersion());
- }
-
-
@Test
public void deleteVerification() throws Throwable {