You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2014/11/06 06:09:56 UTC
git commit: WIP Overwrite
Repository: incubator-usergrid
Updated Branches:
refs/heads/es-type-updates 795874e37 -> 418edde0f
WIP Overwrite
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/418edde0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/418edde0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/418edde0
Branch: refs/heads/es-type-updates
Commit: 418edde0fcb16d7ec1fbf7f31a81a35c3659b020
Parents: 795874e
Author: Todd Nine <tn...@apigee.com>
Authored: Wed Nov 5 22:09:50 2014 -0700
Committer: Todd Nine <tn...@apigee.com>
Committed: Wed Nov 5 22:09:50 2014 -0700
----------------------------------------------------------------------
.../CpEntityIndexDeleteListener.java | 2 +-
.../corepersistence/CpEntityManager.java | 14 +--
.../corepersistence/CpRelationManager.java | 26 +++---
.../results/FilteringLoader.java | 2 +-
.../corepersistence/StaleIndexCleanupTest.java | 2 +-
.../usergrid/persistence/index/EntityIndex.java | 4 +-
.../usergrid/persistence/index/SearchType.java | 66 ++++++++++++++
.../index/impl/EsEntityIndexBatchImpl.java | 96 +++++++++++---------
.../index/impl/EsEntityIndexImpl.java | 39 +++++---
.../persistence/index/impl/IndexScopeImpl.java | 14 +--
.../persistence/index/impl/IndexingUtils.java | 18 +++-
.../index/impl/CorePerformanceIT.java | 17 ++--
.../impl/EntityConnectionIndexImplTest.java | 9 +-
.../persistence/index/impl/EntityIndexTest.java | 9 +-
14 files changed, 208 insertions(+), 110 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
index 125b90b..13becc4 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityIndexDeleteListener.java
@@ -65,7 +65,7 @@ public class CpEntityIndexDeleteListener {
final CollectionScope collectionScope = event.getCollectionScope();
final IndexScope indexScope =
- new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName());
+ new IndexScopeImpl(collectionScope.getOwner(), collectionScope.getName(), entityType );
final EntityIndex entityIndex = entityIndexFactory.createEntityIndex(
new ApplicationScopeImpl( collectionScope.getApplication()));
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index daa046c..97bdba8 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -637,7 +637,7 @@ public class CpEntityManager implements EntityManager {
IndexScope indexScope = new IndexScopeImpl(
new SimpleId( uuid, ownerType ),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ) );
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( coll ), entityType );
batch.index( indexScope, entity );
@@ -650,13 +650,13 @@ public class CpEntityManager implements EntityManager {
// deindex from default index scope
IndexScope defaultIndexScope = new IndexScopeImpl(
getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ), entityType );
batch.deindex(defaultIndexScope, entity );
IndexScope allTypesIndexScope = new IndexScopeImpl(
getApplicationScope().getApplication(),
- CpNamingUtils.ALL_TYPES);
+ CpNamingUtils.ALL_TYPES, entityType );
batch.deindex( allTypesIndexScope, entity );
@@ -1048,7 +1048,7 @@ public class CpEntityManager implements EntityManager {
IndexScope defaultIndexScope = new IndexScopeImpl(
getApplicationScope().getApplication(),
- CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ) );
+ CpNamingUtils.getCollectionScopeNameFromEntityType( entityRef.getType() ), entityType );
EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
EntityIndex ei = managerCache.getEntityIndex(getApplicationScope());
@@ -2976,21 +2976,21 @@ public class CpEntityManager implements EntityManager {
// index member into entity collection | type scope
IndexScope collectionIndexScope = new IndexScopeImpl(
collectionEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
batch.index(collectionIndexScope, memberEntity);
// index member into entity | all-types scope
IndexScope entityAllTypesScope = new IndexScopeImpl(
collectionEntity.getId(),
- CpNamingUtils.ALL_TYPES);
+ CpNamingUtils.ALL_TYPES, entityType );
batch.index(entityAllTypesScope, memberEntity);
// index member into application | all-types scope
IndexScope appAllTypesScope = new IndexScopeImpl(
getApplicationScope().getApplication(),
- CpNamingUtils.ALL_TYPES);
+ CpNamingUtils.ALL_TYPES, entityType );
batch.index(appAllTypesScope, memberEntity);
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
index 1e8dcc3..c33c032 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpRelationManager.java
@@ -413,7 +413,7 @@ public class CpRelationManager implements RelationManager {
String collName = CpNamingUtils.getCollectionName( edge.getType() );
indexScope = new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(),
sourceEntity.getType() ),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
}
else {
@@ -421,7 +421,7 @@ public class CpRelationManager implements RelationManager {
indexScope = new IndexScopeImpl( new SimpleId( sourceEntity.getUuid(),
sourceEntity.getType() ),
CpNamingUtils.getConnectionScopeName( cpEntity.getId().getType(),
- connName ) );
+ connName ), entityType );
}
entityIndexBatch.index( indexScope, cpEntity );
@@ -429,7 +429,7 @@ public class CpRelationManager implements RelationManager {
// reindex the entity in the source entity's all-types index
indexScope = new IndexScopeImpl( new SimpleId(
- sourceEntity.getUuid(), sourceEntity.getType() ), CpNamingUtils.ALL_TYPES );
+ sourceEntity.getUuid(), sourceEntity.getType() ), CpNamingUtils.ALL_TYPES, entityType );
entityIndexBatch.index( indexScope, cpEntity );
}
@@ -806,7 +806,7 @@ public class CpRelationManager implements RelationManager {
// remove item from collection index
IndexScope indexScope = new IndexScopeImpl(
cpHeadEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
batch.deindex( indexScope, memberEntity );
@@ -814,7 +814,7 @@ public class CpRelationManager implements RelationManager {
IndexScope itemScope = new IndexScopeImpl(
memberEntity.getId(),
CpNamingUtils.getCollectionScopeNameFromCollectionName(
- Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ));
+ Schema.defaultCollectionName( cpHeadEntity.getId().getType() ) ), entityType );
batch.deindex( itemScope, cpHeadEntity );
@@ -915,7 +915,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
cpHeadEntity.getId(),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
EntityIndex ei = managerCache.getEntityIndex( applicationScope );
@@ -1034,12 +1034,12 @@ public class CpRelationManager implements RelationManager {
// Index the new connection in app|source|type context
IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(),
- CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ) );
+ CpNamingUtils.getConnectionScopeName( connectedEntityRef.getType(), connectionType ), entityType );
batch.index( indexScope, targetEntity );
// Index the new connection in app|scope|all-types context
- IndexScope allTypesIndexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.ALL_TYPES );
+ IndexScope allTypesIndexScope = new IndexScopeImpl( cpHeadEntity.getId(), CpNamingUtils.ALL_TYPES, entityType );
batch.index( allTypesIndexScope, targetEntity );
@@ -1267,14 +1267,14 @@ public class CpRelationManager implements RelationManager {
new SimpleId( connectingEntityRef.getUuid(),
connectingEntityRef.getType() ),
CpNamingUtils.getConnectionScopeName( targetEntity.getId().getType(),
- connectionType ) );
+ connectionType ), entityType );
batch.deindex( indexScope, targetEntity );
// Deindex the connection in app|source|type context
IndexScope allTypesIndexScope = new IndexScopeImpl(
new SimpleId( connectingEntityRef.getUuid(),
connectingEntityRef.getType() ),
- CpNamingUtils.ALL_TYPES );
+ CpNamingUtils.ALL_TYPES, entityType );
batch.deindex( allTypesIndexScope, targetEntity );
@@ -1334,7 +1334,7 @@ public class CpRelationManager implements RelationManager {
scopeName = CpNamingUtils.ALL_TYPES;
}
- IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), scopeName );
+ IndexScope indexScope = new IndexScopeImpl( cpHeadEntity.getId(), scopeName, entityType );
final EntityIndex ei = managerCache.getEntityIndex( applicationScope );
@@ -1423,7 +1423,7 @@ public class CpRelationManager implements RelationManager {
// search across all types of collections of the head-entity
IndexScope indexScope = new IndexScopeImpl(
cpHeadEntity.getId(),
- CpNamingUtils.ALL_TYPES );
+ CpNamingUtils.ALL_TYPES, entityType );
EntityIndex ei = managerCache.getEntityIndex( applicationScope );
@@ -1440,7 +1440,7 @@ public class CpRelationManager implements RelationManager {
IndexScope indexScope = new IndexScopeImpl(
cpHeadEntity.getId(),
CpNamingUtils.getConnectionScopeName( query.getEntityType(),
- query.getConnectionType() ) );
+ query.getConnectionType() ), entityType );
EntityIndex ei = managerCache.getEntityIndex( applicationScope );
logger.debug("Searching connections from the scope {}:{}",
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
index 8ca2211..c35cb33 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/results/FilteringLoader.java
@@ -241,7 +241,7 @@ public class FilteringLoader implements ResultsLoader {
IndexScope indexScope = new IndexScopeImpl(
ownerId,
- CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()));
+ CpNamingUtils.getCollectionScopeNameFromEntityType( candidateResult.getId().getType()), entityType );
batch.deindex( indexScope, candidateResult );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
----------------------------------------------------------------------
diff --git a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
index 906d7b8..4cd77e5 100644
--- a/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
+++ b/stack/core/src/test/java/org/apache/usergrid/corepersistence/StaleIndexCleanupTest.java
@@ -240,7 +240,7 @@ public class StaleIndexCleanupTest extends AbstractCoreIT {
IndexScope is = new IndexScopeImpl(
new SimpleId( em.getApplicationId(), TYPE_APPLICATION),
- CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ));
+ CpNamingUtils.getCollectionScopeNameFromCollectionName( collName ), entityType );
Query rcq = Query.fromQL(query);
rcq.setLimit(10000); // no paging
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
index 8ded788..bc192dc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/EntityIndex.java
@@ -44,7 +44,7 @@ public interface EntityIndex {
/**
* Execute query in Usergrid syntax.
*/
- public CandidateResults search(final IndexScope indexScope, Query query );
+ public CandidateResults search(final IndexScope indexScope, final SearchType searchType, Query query );
/**
* Get the candidate results of all versions of the entity for this id.
@@ -67,3 +67,5 @@ public interface EntityIndex {
public Health getIndexHealth();
}
+
+
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
new file mode 100644
index 0000000..69137bf
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/SearchType.java
@@ -0,0 +1,66 @@
+package org.apache.usergrid.persistence.index;/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+
+/**
+ * Interface for getting search types
+ */
+public interface SearchType {
+
+ /**
+ * Get the type used for the search
+ * @return
+ */
+ public String getTypeName();
+
+ /**
+ * Constant that defines searching all types
+ */
+ public static final SearchType ALL_TYPES = new SearchType(){
+
+ @Override
+ public String getTypeName() {
+ return "_all";
+ }
+ };
+
+
+ /**
+ * Class to encapsulate search types
+ */
+ public static class EntitySearchType implements SearchType{
+
+ private final String type;
+
+
+ public EntitySearchType( final String type ) {this.type = type;}
+
+
+ @Override
+ public String getTypeName() {
+ return type;
+ }
+
+ public static SearchType fromType(final String type){
+ return new EntitySearchType(type);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
index ad58fa8..9964d0b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexBatchImpl.java
@@ -62,12 +62,13 @@ import com.google.common.base.Joiner;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ANALYZED_STRING_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.BOOLEAN_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITYID_FIELDNAME;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.ENTITY_CONTEXT;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.GEO_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.NUMBER_PREFIX;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PREFIX;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createCollectionScopeTypeName;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexDocId;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createIndexName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
public class EsEntityIndexBatchImpl implements EntityIndexBatch {
@@ -110,36 +111,33 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
IndexValidationUtils.validateIndexScope( indexScope );
- final String indexType = createCollectionScopeTypeName( indexScope );
+ final String context = createContextName( indexScope );
+ final String entityType = entity.getId().getType();
if ( log.isDebugEnabled() ) {
log.debug( "Indexing entity {}:{} in scope\n app {}\n "
- + "owner {}\n name {}\n type {}", new Object[] {
- entity.getId().getType(),
- entity.getId().getUuid(),
- applicationScope.getApplication(),
- indexScope.getOwner(),
- indexScope.getName(), indexType
- });
+ + "owner {}\n name {}\n type {} \n scope type{}", new Object[] {
+ entity.getId().getType(), entity.getId().getUuid(), applicationScope.getApplication(),
+ indexScope.getOwner(), indexScope.getName(), entityType, context
+ } );
}
ValidationUtils.verifyEntityWrite( entity );
- Map<String, Object> entityAsMap = entityToMap( entity );
+ Map<String, Object> entityAsMap = entityToMap( entity, context );
// need prefix here becuase we index UUIDs as strings
- entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME,
- entity.getId().getUuid().toString().toLowerCase() );
+ entityAsMap.put( STRING_PREFIX + ENTITYID_FIELDNAME, entity.getId().getUuid().toString().toLowerCase() );
// let caller add these fields if needed
// entityAsMap.put("created", entity.getId().getUuid().timestamp();
// entityAsMap.put("updated", entity.getVersion().timestamp());
- String indexId = createIndexDocId( entity );
+ String indexId = createIndexDocId( entity, context );
log.debug( "Indexing entity id {} data {} ", indexId, entityAsMap );
- bulkRequest.add(client.prepareIndex( indexName, indexType, indexId).setSource(entityAsMap));
+ bulkRequest.add( client.prepareIndex( indexName, entityType, indexId ).setSource( entityAsMap ) );
maybeFlush();
@@ -152,23 +150,21 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
IndexValidationUtils.validateIndexScope( indexScope );
- final String indexType = createCollectionScopeTypeName( indexScope );
+ final String context = createContextName( indexScope );
+ final String entityType = id.getType();
+
if ( log.isDebugEnabled() ) {
- log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n "
- + "name {} type {}", new Object[] {
- id.getType(),
- id.getUuid(),
- applicationScope.getApplication(),
- indexScope.getOwner(),
- indexScope.getName(),
- indexType
- } );
+ log.debug( "De-indexing entity {}:{} in scope\n app {}\n owner {}\n " + "name {} context{}, type {}",
+ new Object[] {
+ id.getType(), id.getUuid(), applicationScope.getApplication(), indexScope.getOwner(),
+ indexScope.getName(), context, entityType
+ } );
}
- String indexId = createIndexDocId( id, version );
+ String indexId = createIndexDocId( id, version, context );
- bulkRequest.add( client.prepareDelete( indexName, indexType, indexId ).setRefresh(refresh));
+ bulkRequest.add( client.prepareDelete( indexName, entityType, indexId ).setRefresh( refresh ) );
log.debug( "Deindexed Entity with index id " + indexId );
@@ -204,7 +200,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
private void execute( final BulkRequestBuilder request ) {
//nothing to do, we haven't added anthing to the index
- if(request.numberOfActions() == 0){
+ if ( request.numberOfActions() == 0 ) {
return;
}
@@ -212,7 +208,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
try {
responses = request.execute().actionGet();
- }catch(Throwable t){
+ }
+ catch ( Throwable t ) {
log.error( "Unable to communicate with elasticsearch" );
failureMonitor.fail( "Unable to execute batch", t );
throw t;
@@ -223,8 +220,8 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
for ( BulkItemResponse response : responses ) {
if ( response.isFailed() ) {
- throw new RuntimeException("Unable to index documents. Errors are :"
- + response.getFailure().getMessage() );
+ throw new RuntimeException(
+ "Unable to index documents. Errors are :" + response.getFailure().getMessage() );
}
}
@@ -249,11 +246,27 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
/**
+ * Set the entity as a map with the context
+ * @param entity The entity
+ * @param context The context this entity appears in
+ * @return
+ */
+ private static Map entityToMap( final EntityObject entity, final String context ) {
+ final Map entityMap = entityToMap( entity );
+
+ //add the context for filtering later
+ entityMap.put( ENTITY_CONTEXT, context );
+
+ return entityMap;
+ }
+
+
+ /**
* Convert Entity to Map and Adding prefixes for types:
* <pre>
- * su_ - String unanalyzed field
- * sa_ - String analyzed field
- * go_ - Location field nu_ - Number field
+ * su_ - String unanalyzed field
+ * sa_ - String analyzed field
+ * go_ - Location field nu_ - Number field
* bu_ - Boolean field
* </pre>
*/
@@ -267,8 +280,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
if ( f instanceof ListField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
+ entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
if ( !list.isEmpty() ) {
if ( list.get( 0 ) instanceof String ) {
@@ -281,13 +293,11 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof ArrayField ) {
List list = ( List ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( list ) ) );
+ entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( list ) ) );
}
else if ( f instanceof SetField ) {
Set set = ( Set ) field.getValue();
- entityMap.put( field.getName().toLowerCase(),
- new ArrayList( processCollectionForMap( set ) ) );
+ entityMap.put( field.getName().toLowerCase(), new ArrayList( processCollectionForMap( set ) ) );
}
else if ( f instanceof EntityObjectField ) {
EntityObject eo = ( EntityObject ) field.getValue();
@@ -310,9 +320,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
locMap.put( "lon", locField.getValue().getLongitude() );
entityMap.put( GEO_PREFIX + field.getName().toLowerCase(), locMap );
}
- else if ( f instanceof DoubleField
- || f instanceof FloatField
- || f instanceof IntegerField
+ else if ( f instanceof DoubleField || f instanceof FloatField || f instanceof IntegerField
|| f instanceof LongField ) {
entityMap.put( NUMBER_PREFIX + field.getName().toLowerCase(), field.getValue() );
@@ -323,7 +331,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
else if ( f instanceof UUIDField ) {
- entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
+ entityMap.put( STRING_PREFIX + field.getName().toLowerCase(),
field.getValue().toString().toLowerCase() );
}
else {
@@ -335,7 +343,7 @@ public class EsEntityIndexBatchImpl implements EntityIndexBatch {
}
- private static Collection processCollectionForMap( Collection c ) {
+ private static Collection processCollectionForMap( final Collection c ) {
if ( c.isEmpty() ) {
return c;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index efefa8f..d9cfb5b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -26,8 +26,6 @@ import java.util.concurrent.atomic.AtomicBoolean;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
-import org.elasticsearch.action.admin.cluster.health.ClusterHealthStatus;
-import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder;
import org.elasticsearch.action.admin.indices.create.CreateIndexResponse;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexResponse;
import org.elasticsearch.action.admin.indices.template.put.PutIndexTemplateResponse;
@@ -35,14 +33,17 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.client.Client;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermFilterBuilder;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.search.SearchHit;
@@ -53,6 +54,8 @@ import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.lucene.queryparser.xml.FilterBuilderFactory;
+
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
import org.apache.usergrid.persistence.core.util.Health;
import org.apache.usergrid.persistence.core.util.ValidationUtils;
@@ -60,6 +63,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
@@ -228,29 +232,42 @@ public class EsEntityIndexImpl implements EntityIndex {
@Override
- public CandidateResults search( final IndexScope indexScope, final Query query ) {
+ public CandidateResults search( final IndexScope indexScope, final SearchType searchType, final Query query ) {
- final String indexType = IndexingUtils.createCollectionScopeTypeName( indexScope );
+ final String context = IndexingUtils.createContextName( indexScope );
+ final String entityType = searchType.getTypeName();
QueryBuilder qb = query.createQueryBuilder();
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index {}\n type {}\n query {} limit {}", new Object[] {
- this.indexName, indexType, qb.toString().replace( "\n", " " ), query.getLimit()
+ logger.debug( "Searching index {}\n scope{} \n type {}\n query {} limit {}", new Object[] {
+ this.indexName, context, entityType, qb.toString().replace( "\n", " " ), query.getLimit()
} );
}
SearchResponse searchResponse;
if ( query.getCursor() == null ) {
-
SearchRequestBuilder srb =
- esProvider.getClient().prepareSearch( indexName ).setTypes( indexType ).setScroll( cursorTimeout + "m" )
+ esProvider.getClient().prepareSearch( indexName ).setTypes( entityType ).setScroll(
+ cursorTimeout + "m" )
.setQuery( qb );
- FilterBuilder fb = query.createFilterBuilder();
+ final TermFilterBuilder contextFilter = FilterBuilders.termFilter( IndexingUtils.ENTITY_CONTEXT, context );
+
+ final FilterBuilder fb = query.createFilterBuilder();
+
+
+
+ //we have post filters, apply them
if ( fb != null ) {
+ final FilterBuilder postFilters = FilterBuilders.andFilter(fb, contextFilter );
+
logger.debug( " Filter: {} ", fb.toString() );
- srb = srb.setPostFilter( fb );
+ srb = srb.setPostFilter( postFilters );
+ }
+ //no other post filters, just the types
+ else{
+ srb.setPostFilter( contextFilter );
}
srb = srb.setFrom( 0 ).setSize( query.getLimit() );
@@ -384,7 +401,7 @@ public class EsEntityIndexImpl implements EntityIndex {
public CandidateResults getEntityVersions( final IndexScope scope, final Id id ) {
Query query = new Query();
query.addEqualityFilter( ENTITYID_FIELDNAME, id.getUuid().toString() );
- CandidateResults results = search( scope, query );
+ CandidateResults results = search( scope, SearchType.EntitySearchType.fromType( id.getType() ), query );
return results;
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
index 2bf9bb4..13002e3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexScopeImpl.java
@@ -25,12 +25,12 @@ import org.apache.usergrid.persistence.model.entity.Id;
public class IndexScopeImpl implements IndexScope {
private final Id ownerId;
- private final String type;
+ private final String name;
- public IndexScopeImpl( final Id ownerId, final String type ) {
+ public IndexScopeImpl( final Id ownerId, final String name ) {
this.ownerId = ownerId;
- this.type = type;
+ this.name = name;
IndexValidationUtils.validateIndexScope( this );
}
@@ -38,7 +38,7 @@ public class IndexScopeImpl implements IndexScope {
@Override
public String getName() {
- return type;
+ return name;
}
@@ -62,7 +62,7 @@ public class IndexScopeImpl implements IndexScope {
if ( !ownerId.equals( that.ownerId ) ) {
return false;
}
- if ( !type.equals( that.type ) ) {
+ if ( !name.equals( that.name ) ) {
return false;
}
@@ -73,7 +73,7 @@ public class IndexScopeImpl implements IndexScope {
@Override
public int hashCode() {
int result = ownerId.hashCode();
- result = 31 * result + type.hashCode();
+ result = 31 * result + name.hashCode();
return result;
}
@@ -82,7 +82,7 @@ public class IndexScopeImpl implements IndexScope {
public String toString() {
return "IndexScopeImpl{" +
"ownerId=" + ownerId +
- ", type='" + type + '\'' +
+ ", name='" + name + '\'' +
'}';
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index f3e1ba2..d9a3cde 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -48,13 +48,20 @@ public class IndexingUtils {
public static final String INDEX_NAME_SEPARATOR = "^";
+ public static final String ENTITY_CONTEXT = "_context";
+
+ /**
+ * To be used when we want to search all types within a scope
+ */
+ public static final String ALL_TYPES = "ALL";
+
/**
* Create our sub scope. This is the ownerUUID + type
* @param scope
* @return
*/
- public static String createCollectionScopeTypeName( IndexScope scope ) {
+ public static String createContextName( IndexScope scope ) {
StringBuilder sb = new StringBuilder();
sb.append( scope.getOwner().getUuid() ).append(DOC_TYPE_SEPARATOR);
sb.append( scope.getOwner().getType() ).append(DOC_TYPE_SEPARATOR);
@@ -86,8 +93,8 @@ public class IndexingUtils {
* @param entity
* @return
*/
- public static String createIndexDocId(Entity entity) {
- return createIndexDocId(entity.getId(), entity.getVersion());
+ public static String createIndexDocId(final Entity entity, final String scopeType) {
+ return createIndexDocId(entity.getId(), entity.getVersion(), scopeType);
}
@@ -97,11 +104,12 @@ public class IndexingUtils {
* @param version
* @return
*/
- public static String createIndexDocId(Id entityId, UUID version) {
+ public static String createIndexDocId(final Id entityId, final UUID version, final String scopeType) {
StringBuilder sb = new StringBuilder();
sb.append( entityId.getUuid() ).append(DOC_ID_SEPARATOR);
sb.append( entityId.getType() ).append(DOC_ID_SEPARATOR);
- sb.append( version.toString() );
+ sb.append( version.toString() ).append( DOC_ID_SEPARATOR );
+ sb.append( scopeType);
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
index 0d7b0d8..5ec9628 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/CorePerformanceIT.java
@@ -38,6 +38,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.EntityResults;
@@ -127,7 +128,7 @@ public class CorePerformanceIT extends BaseIT {
String appName = "app-" + j + "-" + time;
Id appId = new SimpleId( appName );
- IndexScope indexScope = new IndexScopeImpl( appId, "reviews" );
+ IndexScope indexScope = new IndexScopeImpl( appId, "reviews");
scopes.add( indexScope );
Thread t = new Thread( new DataLoader( applicationScope, indexScope ) );
@@ -178,12 +179,14 @@ public class CorePerformanceIT extends BaseIT {
Query query = Query.fromQL( "review_score > 0"); // get all reviews;
query.withLimit( maxEntities < 1000 ? maxEntities : 1000 );
- CandidateResults candidateResults = eci.search(indexScope, query );
+ final SearchType searchType = SearchType.EntitySearchType.fromType( "review" );
+
+ CandidateResults candidateResults = eci.search(indexScope, searchType, query );
int count = candidateResults.size();
while ( candidateResults.hasCursor() && count < maxEntities ) {
query.setCursor( candidateResults.getCursor() ) ;
- candidateResults = eci.search(indexScope, query );
+ candidateResults = eci.search(indexScope, searchType, query );
count += candidateResults.size();
//cause retrieval from cassandra
@@ -308,10 +311,6 @@ public class CorePerformanceIT extends BaseIT {
public void runSelectedQueries(final ApplicationScope scope, List<IndexScope> indexScopes ) {
for ( IndexScope indexScope : indexScopes ) {
-
-
- CollectionScope collectionScope = new CollectionScopeImpl(
- scope.getApplication(), indexScope.getOwner(), indexScope.getName() );
EntityIndex eci = ecif.createEntityIndex(scope );
// TODO: come up with more and more complex queries for CorePerformanceIT
@@ -332,8 +331,8 @@ public class CorePerformanceIT extends BaseIT {
public static void query(final IndexScope indexScope, final EntityIndex eci, final String query ) {;
Query q = Query.fromQL(query) ;
- CandidateResults candidateResults = eci.search(indexScope, q );
- log.info("size = {} returned from query {}", candidateResults.size(), q.getQl() );
+// CandidateResults candidateResults = eci.search(indexScope, q ); TODO FIXME
+// log.info("size = {} returned from query {}", candidateResults.size(), q.getQl() );
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
index 7ce230b..47bc666 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityConnectionIndexImplTest.java
@@ -40,6 +40,7 @@ import org.apache.usergrid.persistence.index.EntityIndex;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
import org.apache.usergrid.persistence.index.EntityIndexFactory;
import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchType;
import org.apache.usergrid.persistence.index.guice.TestIndexModule;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
@@ -81,9 +82,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
// create a muffin
- CollectionScope muffinScope = new CollectionScopeImpl( appId, appId, "muffins" );
Entity muffin = new Entity(
- new SimpleId( UUIDGenerator.newTimeUUID(), muffinScope.getName() ) );
+ new SimpleId( UUIDGenerator.newTimeUUID(), "muffin" ) );
muffin = EntityIndexMapUtils.fromMap( muffin, new HashMap<String, Object>() {{
put( "size", "Large" );
@@ -93,9 +93,8 @@ public class EntityConnectionIndexImplTest extends BaseIT {
// create a person who likes muffins
- CollectionScope peopleScope = new CollectionScopeImpl( appId, appId, "people" );
Entity person = new Entity( new SimpleId(
- UUIDGenerator.newTimeUUID(), peopleScope.getName() ) );
+ UUIDGenerator.newTimeUUID(), "person" ) );
person = EntityIndexMapUtils.fromMap( person, new HashMap<String, Object>() {{
put( "name", "Dave" );
put( "hometown", "Chapel Hill" );
@@ -118,7 +117,7 @@ public class EntityConnectionIndexImplTest extends BaseIT {
batch.executeAndRefresh();
// now, let's search for things that Dave likes
- CandidateResults likes = personLikesIndex.search(scope, Query.fromQL( "select *" ) );
+ CandidateResults likes = personLikesIndex.search(scope, SearchType.ALL_TYPES, Query.fromQL( "select *" ) );
assertEquals( 1, likes.size() );
assertEquals(muffin.getId(), likes.get(0).getId());
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/418edde0/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 3be996e..e23ad3a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -59,7 +59,6 @@ import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.Maps;
import com.google.inject.Inject;
-import java.util.ArrayList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -152,7 +151,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
- IndexScope indexScope = new IndexScopeImpl( appId, "fastcars" );
+ IndexScope indexScope = new IndexScopeImpl( appId, "fastcars", entityType );
EntityIndex entityIndex = eif.createEntityIndex( applicationScope );
entityIndex.initializeIndex();
@@ -279,7 +278,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
- IndexScope indexScope = new IndexScopeImpl( ownerId, "user" );
+ IndexScope indexScope = new IndexScopeImpl( ownerId, "user", entityType );
@@ -331,7 +330,7 @@ public class EntityIndexTest extends BaseIT {
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
- IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
+ IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
EntityIndex ei = eif.createEntityIndex( applicationScope );
ei.initializeIndex();
@@ -375,7 +374,7 @@ public class EntityIndexTest extends BaseIT {
Id ownerId = new SimpleId( "multivaluedtype" );
ApplicationScope applicationScope = new ApplicationScopeImpl( appId );
- IndexScope appScope = new IndexScopeImpl( ownerId, "user" );
+ IndexScope appScope = new IndexScopeImpl( ownerId, "user", entityType );
EntityIndex ei = eif.createEntityIndex( applicationScope );
ei.createBatch();