You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by sf...@apache.org on 2015/03/25 21:45:08 UTC
incubator-usergrid git commit: change entityindex interface
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-501 e5eb57fd8 -> 8a62d1177
change entityindex interface
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/8a62d117
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/8a62d117
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/8a62d117
Branch: refs/heads/USERGRID-501
Commit: 8a62d1177783612189be49fb924dfead3f348e2f
Parents: e5eb57f
Author: Shawn Feldman <sf...@apache.org>
Authored: Wed Mar 25 14:26:10 2015 -0600
Committer: Shawn Feldman <sf...@apache.org>
Committed: Wed Mar 25 14:26:10 2015 -0600
----------------------------------------------------------------------
.../corepersistence/CpEntityManager.java | 122 ++++++------
.../index/ApplicationEntityIndex.java | 13 +-
.../impl/EsApplicationEntityIndexImpl.java | 199 +++++++------------
.../impl/SearchRequestBuilderStrategy.java | 197 ++++++++++++++++++
.../index/query/CandidateResults.java | 24 ++-
.../usergrid/persistence/index/query/Query.java | 160 +--------------
.../persistence/index/impl/EntityIndexTest.java | 28 +--
.../persistence/query/tree/GrammarTreeTest.java | 8 +-
8 files changed, 382 insertions(+), 369 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
index f420d7b..481fff5 100644
--- a/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
+++ b/stack/core/src/main/java/org/apache/usergrid/corepersistence/CpEntityManager.java
@@ -753,14 +753,14 @@ public class CpEntityManager implements EntityManager {
@Override
public void updateApplication( Application app ) throws Exception {
- update( app );
+ update(app);
this.application = app;
}
@Override
public void updateApplication( Map<String, Object> properties ) throws Exception {
- this.updateProperties( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), properties );
+ this.updateProperties(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), properties);
this.application = get( applicationId, Application.class );
}
@@ -857,7 +857,7 @@ public class CpEntityManager implements EntityManager {
@Override
public EntityRef getAlias( String aliasType, String alias ) throws Exception {
- return getAlias( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ), aliasType, alias );
+ return getAlias(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId), aliasType, alias);
}
@@ -886,7 +886,7 @@ public class CpEntityManager implements EntityManager {
new Object[] { ownerRef, collectionType, aliasValue } );
}
- return results.get( aliasValue );
+ return results.get(aliasValue);
}
@@ -936,7 +936,7 @@ public class CpEntityManager implements EntityManager {
}
- return Collections.<EntityRef>singleton( new SimpleEntityRef( id.getType(), id.getUuid() ) );
+ return Collections.<EntityRef>singleton(new SimpleEntityRef(id.getType(), id.getUuid()));
}
@@ -1003,7 +1003,7 @@ public class CpEntityManager implements EntityManager {
public void setProperty(
EntityRef entityRef, String propertyName, Object propertyValue ) throws Exception {
- setProperty( entityRef, propertyName, propertyValue, false );
+ setProperty(entityRef, propertyName, propertyValue, false);
}
@@ -1021,7 +1021,7 @@ public class CpEntityManager implements EntityManager {
entity.getType(), propertyName, propertyValue );
entity.setProperty( propertyName, propertyValue );
- entity.setProperty( PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis( UUIDUtils.newTimeUUID() ) );
+ entity.setProperty(PROPERTY_MODIFIED, UUIDUtils.getTimestampInMillis(UUIDUtils.newTimeUUID()));
update( entity );
}
@@ -1127,7 +1127,7 @@ public class CpEntityManager implements EntityManager {
public void addToDictionary( EntityRef entityRef, String dictionaryName,
Object elementValue ) throws Exception {
- addToDictionary( entityRef, dictionaryName, elementValue, null );
+ addToDictionary(entityRef, dictionaryName, elementValue, null);
}
@@ -1188,7 +1188,7 @@ public class CpEntityManager implements EntityManager {
EntityRef entity = get( entityRef );
UUID timestampUuid = UUIDUtils.newTimeUUID();
- Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
+ Mutator<ByteBuffer> batch = createMutator(cass.getApplicationKeyspace(applicationId), be);
for ( Map.Entry<?, ?> elementValue : elementValues.entrySet() ) {
batch = batchUpdateDictionary( batch, entity, dictionaryName, elementValue.getKey(),
@@ -1225,8 +1225,8 @@ public class CpEntityManager implements EntityManager {
boolean coTypeIsBasic = ClassUtils.isBasicType( setCoType );
List<HColumn<ByteBuffer, ByteBuffer>> results =
- cass.getAllColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), be, be );
+ cass.getAllColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), be, be);
for ( HColumn<ByteBuffer, ByteBuffer> result : results ) {
Object name = null;
if ( entityHasDictionary ) {
@@ -1275,7 +1275,7 @@ public class CpEntityManager implements EntityManager {
ApplicationCF dictionaryCf = null;
- boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary( entity.getType(), dictionaryName );
+ boolean entityHasDictionary = Schema.getDefaultSchema().hasDictionary(entity.getType(), dictionaryName);
if ( entityHasDictionary ) {
dictionaryCf = ENTITY_DICTIONARIES;
@@ -1285,14 +1285,14 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
+ Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
HColumn<ByteBuffer, ByteBuffer> result =
- cass.getColumn( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ),
- entityHasDictionary ? bytebuffer( elementName ) : DynamicComposite.toByteBuffer( elementName ),
- be, be );
+ cass.getColumn(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName),
+ entityHasDictionary ? bytebuffer(elementName) : DynamicComposite.toByteBuffer(elementName),
+ be, be);
if ( result != null ) {
if ( entityHasDictionary && coTypeIsBasic ) {
@@ -1303,7 +1303,7 @@ public class CpEntityManager implements EntityManager {
}
}
else {
- logger.info( "Results of CpEntityManagerImpl.getDictionaryElementValue is null" );
+ logger.info("Results of CpEntityManagerImpl.getDictionaryElementValue is null");
}
return value;
@@ -1327,8 +1327,8 @@ public class CpEntityManager implements EntityManager {
}
Class<?> dictionaryCoType =
- Schema.getDefaultSchema().getDictionaryValueType( entity.getType(), dictionaryName );
- boolean coTypeIsBasic = ClassUtils.isBasicType( dictionaryCoType );
+ Schema.getDefaultSchema().getDictionaryValueType(entity.getType(), dictionaryName);
+ boolean coTypeIsBasic = ClassUtils.isBasicType(dictionaryCoType);
ByteBuffer[] columnNames = new ByteBuffer[elementNames.length];
for ( int i = 0; i < elementNames.length; i++ ) {
@@ -1337,8 +1337,8 @@ public class CpEntityManager implements EntityManager {
}
ColumnSlice<ByteBuffer, ByteBuffer> results =
- cass.getColumns( cass.getApplicationKeyspace( applicationId ), dictionaryCf,
- CassandraPersistenceUtils.key( entity.getUuid(), dictionaryName ), columnNames, be, be );
+ cass.getColumns(cass.getApplicationKeyspace(applicationId), dictionaryCf,
+ CassandraPersistenceUtils.key(entity.getUuid(), dictionaryName), columnNames, be, be);
if ( results != null ) {
values = new HashMap<String, Object>();
for ( HColumn<ByteBuffer, ByteBuffer> result : results.getColumns() ) {
@@ -1384,7 +1384,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getDictionaries( EntityRef entity ) throws Exception {
- return getDictionaryNames( entity );
+ return getDictionaryNames(entity);
}
@@ -1398,14 +1398,14 @@ public class CpEntityManager implements EntityManager {
@Override
public boolean isCollectionMember( EntityRef owner, String collectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isCollectionMember( collectionName, entity );
+ return getRelationManager( owner ).isCollectionMember(collectionName, entity);
}
@Override
public boolean isConnectionMember( EntityRef owner, String connectionName, EntityRef entity ) throws Exception {
- return getRelationManager( owner ).isConnectionMember( connectionName, entity );
+ return getRelationManager( owner ).isConnectionMember(connectionName, entity);
}
@@ -1430,7 +1430,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( get( entityId ))
- .getCollection ( collectionName, query, resultsLevel );
+ .getCollection(collectionName, query, resultsLevel);
}
@@ -1495,8 +1495,8 @@ public class CpEntityManager implements EntityManager {
@Override
public ConnectionRef createConnection( ConnectionRef connection ) throws Exception {
- return createConnection( connection.getConnectingEntity(), connection.getConnectionType(),
- connection.getConnectedEntity() );
+ return createConnection(connection.getConnectingEntity(), connection.getConnectionType(),
+ connection.getConnectedEntity());
}
@@ -1504,7 +1504,7 @@ public class CpEntityManager implements EntityManager {
public ConnectionRef createConnection( EntityRef connectingEntity, String connectionType,
EntityRef connectedEntityRef ) throws Exception {
- return getRelationManager( connectingEntity ).createConnection( connectionType, connectedEntityRef );
+ return getRelationManager( connectingEntity ).createConnection(connectionType, connectedEntityRef);
}
@@ -1514,7 +1514,7 @@ public class CpEntityManager implements EntityManager {
throws Exception {
return getRelationManager( connectingEntity )
- .createConnection( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
+ .createConnection(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
}
@@ -1540,7 +1540,7 @@ public class CpEntityManager implements EntityManager {
EntityRef pairedEntity, String connectionType, EntityRef connectedEntityRef ) throws Exception {
return getRelationManager( connectingEntity )
- .connectionRef( pairedConnectionType, pairedEntity, connectionType, connectedEntityRef );
+ .connectionRef(pairedConnectionType, pairedEntity, connectionType, connectedEntityRef);
}
@@ -1572,7 +1572,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectedEntities( connectionType, connectedEntityType, resultsLevel );
+ .getConnectedEntities(connectionType, connectedEntityType, resultsLevel);
}
@@ -1581,7 +1581,7 @@ public class CpEntityManager implements EntityManager {
String connectedEntityType, Level resultsLevel ) throws Exception {
return getRelationManager( entityRef )
- .getConnectingEntities( connectionType, connectedEntityType, resultsLevel );
+ .getConnectingEntities(connectionType, connectedEntityType, resultsLevel);
}
@@ -1589,21 +1589,21 @@ public class CpEntityManager implements EntityManager {
public Results getConnectingEntities( EntityRef entityRef, String connectionType,
String entityType, Level level, int count ) throws Exception {
- return getRelationManager( entityRef ).getConnectingEntities( connectionType, entityType, level, count );
+ return getRelationManager( entityRef ).getConnectingEntities(connectionType, entityType, level, count);
}
@Override
public Results searchConnectedEntities( EntityRef connectingEntity, Query query ) throws Exception {
- return getRelationManager( connectingEntity ).searchConnectedEntities( query );
+ return getRelationManager( connectingEntity ).searchConnectedEntities(query);
}
@Override
public Set<String> getConnectionIndexes( EntityRef entity, String connectionType ) throws Exception {
- return getRelationManager( entity ).getConnectionIndexes( connectionType );
+ return getRelationManager( entity ).getConnectionIndexes(connectionType);
}
@@ -1829,14 +1829,14 @@ public class CpEntityManager implements EntityManager {
String roleTitle = batchRoleName;
String propertyName = groupId + ":" + batchRoleName;
Map<String, Object> properties = new TreeMap<String, Object>( CASE_INSENSITIVE_ORDER );
- properties.put( "group", groupId );
+ properties.put("group", groupId);
- Entity entity = batchCreateRole( roleName, roleTitle, inactivity, propertyName, groupId, properties );
+ Entity entity = batchCreateRole(roleName, roleTitle, inactivity, propertyName, groupId, properties);
getRelationManager( new SimpleEntityRef( Group.ENTITY_TYPE, groupId ) )
- .addToCollection( COLLECTION_ROLES, entity );
+ .addToCollection(COLLECTION_ROLES, entity);
- logger.info( "Created role {} with id {} in group {}",
- new String[] { roleName, entity.getUuid().toString(), groupId.toString() } );
+ logger.info("Created role {} with id {} in group {}",
+ new String[]{roleName, entity.getUuid().toString(), groupId.toString()});
return entity;
}
@@ -1848,8 +1848,8 @@ public class CpEntityManager implements EntityManager {
permission = permission.toLowerCase();
long timestamp = cass.createTimestamp();
Mutator<ByteBuffer> batch = createMutator( cass.getApplicationKeyspace( applicationId ), be );
- CassandraPersistenceUtils.addInsertToMutator( batch, ApplicationCF.ENTITY_DICTIONARIES,
- getRolePermissionsKey( groupId, roleName ), permission, ByteBuffer.allocate( 0 ), timestamp );
+ CassandraPersistenceUtils.addInsertToMutator(batch, ApplicationCF.ENTITY_DICTIONARIES,
+ getRolePermissionsKey(groupId, roleName), permission, ByteBuffer.allocate(0), timestamp);
//Adding graphite metrics
Timer.Context timeGroupRolePermission = entGrantGroupPermissionTimer.time();
@@ -1927,7 +1927,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantUserPermission( UUID userId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary( userRef( userId ), DICTIONARY_PERMISSIONS, permission );
+ addToDictionary(userRef(userId), DICTIONARY_PERMISSIONS, permission);
}
@@ -1951,9 +1951,9 @@ public class CpEntityManager implements EntityManager {
roleName = roleName.toLowerCase();
EntityRef userRef = userRef( userId );
EntityRef roleRef = getRoleRef( roleName );
- addToDictionary( userRef, DICTIONARY_ROLENAMES, roleName, roleName );
- addToCollection( userRef, COLLECTION_ROLES, roleRef );
- addToCollection( roleRef, COLLECTION_USERS, userRef );
+ addToDictionary(userRef, DICTIONARY_ROLENAMES, roleName, roleName);
+ addToCollection(userRef, COLLECTION_ROLES, roleRef);
+ addToCollection(roleRef, COLLECTION_USERS, userRef);
}
@@ -1988,8 +1988,8 @@ public class CpEntityManager implements EntityManager {
private EntityRef getRoleRef( String roleName ) throws Exception {
- Results results = this.searchCollection( new SimpleEntityRef( Application.ENTITY_TYPE, applicationId ),
- Schema.defaultCollectionName( Role.ENTITY_TYPE ), Query.findForProperty( "roleName", roleName ) );
+ Results results = this.searchCollection(new SimpleEntityRef(Application.ENTITY_TYPE, applicationId),
+ Schema.defaultCollectionName(Role.ENTITY_TYPE), Query.findForProperty("roleName", roleName));
Iterator<Entity> iterator = results.iterator();
EntityRef roleRef = null;
while ( iterator.hasNext() ) {
@@ -2235,8 +2235,8 @@ public class CpEntityManager implements EntityManager {
@Override
public Set<String> getCounterNames() throws Exception {
Set<String> names = new TreeSet<String>( CASE_INSENSITIVE_ORDER );
- Set<String> nameSet = cast( getDictionaryAsSet( getApplicationRef(), Schema.DICTIONARY_COUNTERS ) );
- names.addAll( nameSet );
+ Set<String> nameSet = cast(getDictionaryAsSet(getApplicationRef(), Schema.DICTIONARY_COUNTERS));
+ names.addAll(nameSet);
return names;
}
@@ -2300,7 +2300,7 @@ public class CpEntityManager implements EntityManager {
CollectionScope collectionScope = getCollectionScopeNameFromEntityType(
getApplicationScope().getApplication(), collectionName);
- final EntityCollectionManager ecm = managerCache.getEntityCollectionManager( collectionScope );
+ final EntityCollectionManager ecm = managerCache.getEntityCollectionManager(collectionScope);
//convert to a string, that's what we store
final Id results = ecm.getIdField( new StringField(
@@ -2313,7 +2313,7 @@ public class CpEntityManager implements EntityManager {
public Entity get( UUID uuid ) throws Exception {
MapManager mm = getMapManagerForTypes();
- String entityType = mm.getString( uuid.toString() );
+ String entityType = mm.getString(uuid.toString());
final Entity entity;
@@ -2335,9 +2335,9 @@ public class CpEntityManager implements EntityManager {
private MapManager getMapManagerForTypes() {
Id mapOwner = new SimpleId( applicationId, TYPE_APPLICATION );
- final MapScope ms = CpNamingUtils.getEntityTypeMapScope( mapOwner );
+ final MapScope ms = CpNamingUtils.getEntityTypeMapScope(mapOwner);
- MapManager mm = managerCache.getMapManager( ms );
+ MapManager mm = managerCache.getMapManager(ms);
return mm;
}
@@ -2440,7 +2440,7 @@ public class CpEntityManager implements EntityManager {
@Override
public Map<String, Role> getUserRolesWithTitles( UUID userId ) throws Exception {
return getRolesWithTitles(
- ( Set<String> ) cast( getDictionaryAsSet( userRef( userId ), DICTIONARY_ROLENAMES ) ) );
+ ( Set<String> ) cast( getDictionaryAsSet( userRef(userId), DICTIONARY_ROLENAMES ) ) );
}
@@ -2455,8 +2455,8 @@ public class CpEntityManager implements EntityManager {
@Override
public void addGroupToRole( UUID groupId, String roleName ) throws Exception {
roleName = roleName.toLowerCase();
- addToDictionary( groupRef( groupId ), DICTIONARY_ROLENAMES, roleName, roleName );
- addToCollection( groupRef( groupId ), COLLECTION_ROLES, getRoleRef( roleName ) );
+ addToDictionary(groupRef(groupId), DICTIONARY_ROLENAMES, roleName, roleName);
+ addToCollection(groupRef(groupId), COLLECTION_ROLES, getRoleRef(roleName));
}
@@ -2477,7 +2477,7 @@ public class CpEntityManager implements EntityManager {
@Override
public void grantGroupPermission( UUID groupId, String permission ) throws Exception {
permission = permission.toLowerCase();
- addToDictionary( groupRef( groupId ), DICTIONARY_PERMISSIONS, permission );
+ addToDictionary(groupRef(groupId), DICTIONARY_PERMISSIONS, permission);
}
@@ -2663,8 +2663,8 @@ public class CpEntityManager implements EntityManager {
catch ( HystrixRuntimeException hre ) {
if ( hre.getCause() instanceof WriteUniqueVerifyException ) {
- WriteUniqueVerifyException wuve = ( WriteUniqueVerifyException ) hre.getCause();
- handleWriteUniqueVerifyException( entity, wuve );
+ WriteUniqueVerifyException wuve = (WriteUniqueVerifyException) hre.getCause();
+ handleWriteUniqueVerifyException(entity, wuve);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
index dd1c5fa..2167efa 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/ApplicationEntityIndex.java
@@ -39,11 +39,22 @@ public interface ApplicationEntityIndex {
*/
public EntityIndexBatch createBatch();
+ /**
+ * Execute query in Usergrid syntax.
+ */
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query);
/**
* Execute query in Usergrid syntax.
*/
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, Query query );
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchType, final Query query, final int limit );
+
+ /**
+ * get next page of results
+ * @param cursor
+ * @return
+ */
+ public CandidateResults getNextPage(final String cursor);
/**
* delete all application records
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index ade2286..ca071e2 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -46,15 +46,9 @@ import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.TermQueryBuilder;
+import org.elasticsearch.index.query.*;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
-import org.elasticsearch.search.sort.FieldSortBuilder;
-import org.elasticsearch.search.sort.SortBuilders;
-import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import rx.Observable;
@@ -85,6 +79,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
private final IndexIdentifier.IndexAlias alias;
private final Timer deleteApplicationTimer;
private final Meter deleteApplicationMeter;
+ private final SearchRequestBuilderStrategy searchRequest;
private FailureMonitor failureMonitor;
private final int cursorTimeout;
@Inject
@@ -117,6 +112,8 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
this.alias = indexIdentifier.getAlias();
+ this.searchRequest = new SearchRequestBuilderStrategy(esProvider,appScope,alias,cursorTimeout);
+
}
@Override
@@ -139,132 +136,78 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
}
@Override
- public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes,
- final Query query ) {
-
- SearchResponse searchResponse;
-
- if ( query.getCursor() == null ) {
- String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
- .setTypes(searchTypes.getTypeNames(applicationScope))
- .setScroll(cursorTimeout + "m")
- .setQuery(query.createQueryBuilder(contexts));
-
- final FilterBuilder fb = query.createFilterBuilder();
-
- //we have post filters, apply them
- if ( fb != null ) {
- logger.debug( " Filter: {} ", fb.toString() );
- srb = srb.setPostFilter( fb );
- }
-
-
- srb = srb.setFrom( 0 ).setSize( query.getLimit() );
-
- for ( Query.SortPredicate sp : query.getSortPredicates() ) {
-
- final SortOrder order;
- if ( sp.getDirection().equals( Query.SortDirection.ASCENDING ) ) {
- order = SortOrder.ASC;
- }
- else {
- order = SortOrder.DESC;
- }
-
- // we do not know the type of the "order by" property and so we do not know what
- // type prefix to use. So, here we add an order by clause for every possible type
- // that you can order by: string, number and boolean and we ask ElasticSearch
- // to ignore any fields that are not present.
-
- final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
- .order( order ).ignoreUnmapped( true );
- srb.addSort( stringSort );
-
- logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query){
+ return search(indexScope,searchTypes,query,10);
+ }
- final String longFieldName = LONG_PREFIX + sp.getPropertyName();
- final FieldSortBuilder longSort = SortBuilders.fieldSort( longFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( longSort );
- logger.debug( " Sort: {} order by {}", longFieldName, order.toString() );
+ @Override
+ public CandidateResults search(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ SearchResponse searchResponse;
- final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
- final FieldSortBuilder doubleSort = SortBuilders.fieldSort( doubleFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( doubleSort );
- logger.debug( " Sort: {} order by {}", doubleFieldName, order.toString() );
+ SearchRequestBuilder srb = searchRequest.getBuilder(indexScope, searchTypes, query, limit);
+ if (logger.isDebugEnabled()) {
+ logger.debug("Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
+ this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb);
+ }
- final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( booleanSort );
- logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
- }
+ try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearch = searchTimer.time();
+ searchResponse = srb.execute().actionGet();
+ timeSearch.stop();
+ } catch (Throwable t) {
+ logger.error("Unable to communicate with Elasticsearch", t);
+ failureMonitor.fail("Unable to execute batch", t);
+ throw t;
+ }
+ failureMonitor.success();
- if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
- this.alias.getReadAlias(), indexScope.getOwner(), searchTypes.getTypeNames(applicationScope), srb );
- }
+ return parseResults(searchResponse);
+ }
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearch = searchTimer.time();
- searchResponse = srb.execute().actionGet();
- timeSearch.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with Elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
+ public CandidateResults getNextPage(final String cursor){
+ SearchResponse searchResponse;
- failureMonitor.success();
+ String userCursorString = cursor;
+ if ( userCursorString.startsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 1 );
+ }
+ if ( userCursorString.endsWith( "\"" ) ) {
+ userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
}
- else {
- String userCursorString = query.getCursor();
- if ( userCursorString.startsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 1 );
- }
- if ( userCursorString.endsWith( "\"" ) ) {
- userCursorString = userCursorString.substring( 0, userCursorString.length() - 1 );
- }
-
- //now get the cursor from the map and validate
- final String esScrollCursor = mapManager.getString( userCursorString );
-
- Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
+ //now get the cursor from the map and validate
+ final String esScrollCursor = mapManager.getString(userCursorString);
+ Preconditions.checkArgument(esScrollCursor != null, "Could not find a cursor for the value '{}' ", esScrollCursor);
- logger.debug( "Executing query with cursor: {} ", esScrollCursor );
- SearchScrollRequestBuilder ssrb = esProvider.getClient()
- .prepareSearchScroll(esScrollCursor).setScroll( cursorTimeout + "m" );
+ logger.debug( "Executing query with cursor: {} ", esScrollCursor );
- try {
- //Added For Graphite Metrics
- Timer.Context timeSearchCursor = cursorTimer.time();
- searchResponse = ssrb.execute().actionGet();
- timeSearchCursor.stop();
- }
- catch ( Throwable t ) {
- logger.error( "Unable to communicate with elasticsearch", t );
- failureMonitor.fail( "Unable to execute batch", t );
- throw t;
- }
+ SearchScrollRequestBuilder ssrb = esProvider.getClient()
+ .prepareSearchScroll(esScrollCursor).setScroll(cursorTimeout + "m");
- failureMonitor.success();
+ try {
+ //Added For Graphite Metrics
+ Timer.Context timeSearchCursor = cursorTimer.time();
+ searchResponse = ssrb.execute().actionGet();
+ timeSearchCursor.stop();
+ }
+ catch ( Throwable t ) {
+ logger.error( "Unable to communicate with elasticsearch", t );
+ failureMonitor.fail( "Unable to execute batch", t );
+ throw t;
}
- return parseResults(searchResponse, query);
+
+ failureMonitor.success();
+ return parseResults(searchResponse);
}
/**
@@ -327,7 +270,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
- private CandidateResults parseResults( final SearchResponse searchResponse, final Query query ) {
+ private CandidateResults parseResults( final SearchResponse searchResponse) {
final SearchHits searchHits = searchResponse.getHits();
final SearchHit[] hits = searchHits.getHits();
@@ -335,41 +278,39 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex{
logger.debug(" Hit count: {} Total hits: {}", length, searchHits.getTotalHits());
- List<CandidateResult> candidates = new ArrayList<>( length );
+ List<CandidateResult> candidates = new ArrayList<>(length);
- for ( SearchHit hit : hits ) {
+ for (SearchHit hit : hits) {
- String[] idparts = hit.getId().split( SPLITTER );
+ String[] idparts = hit.getId().split(SPLITTER);
String id = idparts[0];
String type = idparts[1];
String version = idparts[2];
- Id entityId = new SimpleId( UUID.fromString(id), type );
+ Id entityId = new SimpleId(UUID.fromString(id), type);
- candidates.add( new CandidateResult( entityId, UUID.fromString( version ) ) );
+ candidates.add(new CandidateResult(entityId, UUID.fromString(version)));
}
- CandidateResults candidateResults = new CandidateResults( query, candidates );
-
- if ( candidates.size() >= query.getLimit() ) {
- //USERGRID-461 our cursor is getting too large, map it to a new time UUID
- //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home
-
- final String userCursorString = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID( UUIDGenerator.newTimeUUID() );
+ CandidateResults candidateResults = new CandidateResults(candidates);
+ if(candidateResults.hasCursor()) {
+ candidateResults.initializeCursor();
final String esScrollCursor = searchResponse.getScrollId();
-
//now set this into our map module
final int minutes = indexFig.getQueryCursorTimeout();
//just truncate it, we'll never hit a long value anyway
- mapManager.putString( userCursorString, esScrollCursor, ( int ) TimeUnit.MINUTES.toSeconds( minutes ) );
+ mapManager.putString(candidateResults.getCursor(), esScrollCursor, (int) TimeUnit.MINUTES.toSeconds(minutes));
- candidateResults.setCursor( userCursorString );
- logger.debug(" User cursor = {}, Cursor = {} ", userCursorString, esScrollCursor);
+ logger.debug(" User cursor = {}, Cursor = {} ", candidateResults.getCursor(), esScrollCursor);
}
return candidateResults;
}
+
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
new file mode 100644
index 0000000..d146e4c
--- /dev/null
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -0,0 +1,197 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one or more
+ * * contributor license agreements. The ASF licenses this file to You
+ * * under the Apache License, Version 2.0 (the "License"); you may not
+ * * use this file except in compliance with the License.
+ * * You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing, software
+ * * distributed under the License is distributed on an "AS IS" BASIS,
+ * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * * See the License for the specific language governing permissions and
+ * * limitations under the License. For additional information regarding
+ * * copyright in this work, please see the NOTICE file in the top level
+ * * directory of this distribution.
+ *
+ */
+package org.apache.usergrid.persistence.index.impl;
+
+import com.google.common.base.Preconditions;
+import org.apache.usergrid.persistence.core.scope.ApplicationScope;
+import org.apache.usergrid.persistence.core.util.ValidationUtils;
+import org.apache.usergrid.persistence.index.IndexIdentifier;
+import org.apache.usergrid.persistence.index.IndexScope;
+import org.apache.usergrid.persistence.index.SearchTypes;
+import org.apache.usergrid.persistence.index.exceptions.IndexException;
+import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
+import org.elasticsearch.action.search.SearchRequestBuilder;
+import org.elasticsearch.index.query.BoolQueryBuilder;
+import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.FieldSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
+import org.elasticsearch.search.sort.SortOrder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createLegacyContextName;
+
+/**
+ * Classy class class.
+ */
+
+public class SearchRequestBuilderStrategy {
+
+ private static final Logger logger = LoggerFactory.getLogger(SearchRequestBuilderStrategy.class);
+
+ private final EsProvider esProvider;
+ private final ApplicationScope applicationScope;
+ private final IndexIdentifier.IndexAlias alias;
+ private final int cursorTimeout;
+ public static final int MAX_LIMIT = 1000;
+
+ public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexIdentifier.IndexAlias alias, int cursorTimeout){
+
+ this.esProvider = esProvider;
+ this.applicationScope = applicationScope;
+ this.alias = alias;
+ this.cursorTimeout = cursorTimeout;
+ }
+
+ public SearchRequestBuilder getBuilder(final IndexScope indexScope, final SearchTypes searchTypes, final Query query, final int limit) {
+ Preconditions.checkArgument(limit <= MAX_LIMIT, "limit is greater than max "+ MAX_LIMIT);
+
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
+ .setTypes(searchTypes.getTypeNames(applicationScope))
+ .setScroll(cursorTimeout + "m")
+ .setQuery(createQueryBuilder( indexScope,query));
+
+ final FilterBuilder fb = createFilterBuilder(query);
+
+ //we have post filters, apply them
+ if (fb != null) {
+ logger.debug(" Filter: {} ", fb.toString());
+ srb = srb.setPostFilter(fb);
+ }
+
+
+ srb = srb.setFrom(0).setSize(limit);
+
+ for (Query.SortPredicate sp : query.getSortPredicates()) {
+
+ final SortOrder order;
+ if (sp.getDirection().equals(Query.SortDirection.ASCENDING)) {
+ order = SortOrder.ASC;
+ } else {
+ order = SortOrder.DESC;
+ }
+
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
+
+ final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(stringSort);
+
+ logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
+
+ final String longFieldName = LONG_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(longSort);
+ logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
+
+
+ final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(doubleSort);
+ logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
+
+
+ final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
+ .order(order).ignoreUnmapped(true);
+ srb.addSort(booleanSort);
+ logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+ }
+ return srb;
+ }
+
+
+ public QueryBuilder createQueryBuilder( IndexScope indexScope, Query query) {
+ String[] contexts = new String[]{createContextName(applicationScope, indexScope),createLegacyContextName(applicationScope,indexScope)};
+
+ QueryBuilder queryBuilder = null;
+
+ //we have a root operand. Translate our AST into an ES search
+ if ( query.getRootOperand() != null ) {
+ // In the case of geo only queries, this will return null into the query builder.
+ // Once we start using tiles, we won't need this check any longer, since a geo query
+ // will return a tile query + post filter
+ QueryVisitor v = new EsQueryVistor();
+
+ try {
+ query.getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+
+
+ queryBuilder = v.getQueryBuilder();
+ }
+
+
+ // Add our filter for context to our query for fast execution.
+ // Fast because it utilizes bitsets internally. See this post for more detail.
+ // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
+
+ // TODO evaluate performance when it's an all query.
+ // Do we need to put the context term first for performance?
+ BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
+ for(String context : contexts){
+ boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
+ }
+ boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
+ if ( queryBuilder != null ) {
+ queryBuilder = boolQueryBuilder.must( queryBuilder );
+ }
+
+ //nothing was specified ensure we specify the context in the search
+ else {
+ queryBuilder = boolQueryBuilder;
+ }
+
+ return queryBuilder;
+ }
+
+
+ public FilterBuilder createFilterBuilder(Query query) {
+ FilterBuilder filterBuilder = null;
+
+ if ( query.getRootOperand() != null ) {
+ QueryVisitor v = new EsQueryVistor();
+ try {
+ query.getRootOperand().visit( v );
+
+ } catch ( IndexException ex ) {
+ throw new RuntimeException( "Error building ElasticSearch query", ex );
+ }
+ filterBuilder = v.getFilterBuilder();
+ }
+
+ return filterBuilder;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
index be90446..5cb6680 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/CandidateResults.java
@@ -19,12 +19,14 @@ package org.apache.usergrid.persistence.index.query;
import java.util.Iterator;
import java.util.List;
+
+import org.apache.usergrid.persistence.model.util.UUIDGenerator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Internal results class, should not be returned as results to a user.
+ * Internal results class, should not be returned as results to a user.
* Only returns candidate entity results
*/
public class CandidateResults implements Iterable<CandidateResult> {
@@ -33,16 +35,25 @@ public class CandidateResults implements Iterable<CandidateResult> {
private String cursor = null;
- private final Query query;
private final List<CandidateResult> candidates;
- public CandidateResults( Query query, List<CandidateResult> candidates ) {
- this.query = query;
+ public CandidateResults( List<CandidateResult> candidates ) {
this.candidates = candidates;
}
+ public void add( List<CandidateResult> candidates ) {
+ this.candidates.addAll( candidates);
+ }
+
+ public void initializeCursor(){
+ //USERGRID-461 our cursor is getting too large, map it to a new time UUID
+ //TODO T.N., this shouldn't live here. This should live at the UG core tier. However the RM/EM are an absolute mess, so until they're refactored, this is it's home
+
+ cursor = org.apache.usergrid.persistence.index.utils.StringUtils.sanitizeUUID(UUIDGenerator.newTimeUUID());
+ }
+
public boolean hasCursor() {
return cursor != null;
@@ -59,11 +70,8 @@ public class CandidateResults implements Iterable<CandidateResult> {
}
- public Query getQuery() {
- return query;
- }
-
+
public int size() {
return candidates.size();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index 438d82e..6454b48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -40,10 +40,7 @@ import org.antlr.runtime.Token;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.lang.StringUtils;
-import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
-import org.apache.usergrid.persistence.index.impl.EsQueryVistor;
-import org.apache.usergrid.persistence.index.impl.IndexingUtils;
import org.apache.usergrid.persistence.index.query.tree.AndOperand;
import org.apache.usergrid.persistence.index.query.tree.ContainsOperand;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
@@ -55,15 +52,10 @@ import org.apache.usergrid.persistence.index.query.tree.GreaterThanEqual;
import org.apache.usergrid.persistence.index.query.tree.LessThan;
import org.apache.usergrid.persistence.index.query.tree.LessThanEqual;
import org.apache.usergrid.persistence.index.query.tree.Operand;
-import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.ClassUtils;
import org.apache.usergrid.persistence.index.utils.ConversionUtils;
import org.apache.usergrid.persistence.index.utils.ListUtils;
import org.apache.usergrid.persistence.index.utils.MapUtils;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -80,9 +72,7 @@ public class Query {
IDS, REFS, CORE_PROPERTIES, ALL_PROPERTIES, LINKED_PROPERTIES
}
- public static final int DEFAULT_LIMIT = 10;
- public static final int MAX_LIMIT = 1000;
public static final String PROPERTY_UUID = "uuid";
@@ -90,8 +80,6 @@ public class Query {
private List<SortPredicate> sortPredicates = new ArrayList<SortPredicate>();
private Operand rootOperand;
private UUID startResult;
- private String cursor;
- private int limit = 0;
private Map<String, String> selectAssignments = new LinkedHashMap<String, String>();
private boolean mergeSelectResults = false;
@@ -124,8 +112,6 @@ public class Query {
sortPredicates = q.sortPredicates != null
? new ArrayList<SortPredicate>( q.sortPredicates ) : null;
startResult = q.startResult;
- cursor = q.cursor;
- limit = q.limit;
selectAssignments = q.selectAssignments != null
? new LinkedHashMap<String, String>( q.selectAssignments ) : null;
mergeSelectResults = q.mergeSelectResults;
@@ -148,71 +134,6 @@ public class Query {
}
- public QueryBuilder createQueryBuilder( final String[] contexts ) {
-
-
- QueryBuilder queryBuilder = null;
-
-
- //we have a root operand. Translate our AST into an ES search
- if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
-
- try {
- getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
-
-
- queryBuilder = v.getQueryBuilder();
- }
-
-
- // Add our filter for context to our query for fast execution.
- // Fast because it utilizes bitsets internally. See this post for more detail.
- // http://www.elasticsearch.org/blog/all-about-elasticsearch-filter-bitsets/
-
- // TODO evaluate performance when it's an all query.
- // Do we need to put the context term first for performance?
- BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
- for(String context : contexts){
- boolQueryBuilder = boolQueryBuilder.should(QueryBuilders.termQuery( IndexingUtils.ENTITY_CONTEXT_FIELDNAME, context ));
- }
- boolQueryBuilder = boolQueryBuilder.minimumNumberShouldMatch(1);
- if ( queryBuilder != null ) {
- queryBuilder = boolQueryBuilder.must( queryBuilder );
- }
-
- //nothing was specified ensure we specify the context in the search
- else {
- queryBuilder = boolQueryBuilder;
- }
-
- return queryBuilder;
- }
-
-
- public FilterBuilder createFilterBuilder() {
- FilterBuilder filterBuilder = null;
-
- if ( getRootOperand() != null ) {
- QueryVisitor v = new EsQueryVistor();
- try {
- getRootOperand().visit( v );
-
- } catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
- filterBuilder = v.getFilterBuilder();
- }
-
- return filterBuilder;
- }
/**
@@ -315,7 +236,6 @@ public class Query {
String connection = ListUtils.first( params.get( "connectionType" ) );
UUID start = ListUtils.firstUuid( params.get( "start" ) );
String cursor = ListUtils.first( params.get( "cursor" ) );
- Integer limit = ListUtils.firstInteger( params.get( "limit" ) );
List<String> permissions = params.get( "permission" );
Long startTime = ListUtils.firstLong( params.get( "start_time" ) );
Long finishTime = ListUtils.firstLong( params.get( "end_time" ) );
@@ -386,15 +306,7 @@ public class Query {
q.setStartResult( start );
}
- if ( cursor != null ) {
- q = newQueryIfNull( q );
- q.setCursor( cursor );
- }
- if ( limit != null ) {
- q = newQueryIfNull( q );
- q.setLimit( limit );
- }
if ( startTime != null ) {
q = newQueryIfNull( q );
@@ -437,22 +349,21 @@ public class Query {
public static Query searchForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter( propertyName, propertyValue );
+ q.addEqualityFilter(propertyName, propertyValue);
return q;
}
public static Query findForProperty( String propertyName, Object propertyValue ) {
Query q = new Query();
- q.addEqualityFilter( propertyName, propertyValue );
- q.setLimit( 1 );
+ q.addEqualityFilter(propertyName, propertyValue);
return q;
}
public static Query fromUUID( UUID uuid ) {
Query q = new Query();
- q.addIdentifier( Identifier.fromUUID( uuid ) );
+ q.addIdentifier( Identifier.fromUUID(uuid) );
return q;
}
@@ -809,8 +720,8 @@ public class Query {
public Query addContainsFilter( String propName, String keyword ) {
ContainsOperand equality = new ContainsOperand( new ClassicToken( 0, "contains" ) );
- equality.setProperty( propName );
- equality.setLiteral( keyword );
+ equality.setProperty(propName);
+ equality.setLiteral(keyword);
addClause( equality );
@@ -819,8 +730,8 @@ public class Query {
private void addClause( EqualityOperand equals, String propertyName, Object value ) {
- equals.setProperty( propertyName );
- equals.setLiteral( value );
+ equals.setProperty(propertyName);
+ equals.setLiteral(value);
addClause( equals );
}
@@ -882,7 +793,7 @@ public class Query {
}
- public UUID getStartResult() {
+ public UUID getStartResult(String cursor) {
if ( ( startResult == null ) && ( cursor != null ) ) {
byte[] cursorBytes = Base64.decodeBase64( cursor );
if ( ( cursorBytes != null ) && ( cursorBytes.length == 16 ) ) {
@@ -893,61 +804,6 @@ public class Query {
}
- public String getCursor() {
- return cursor;
- }
-
-
- public void setCursor( String cursor ) {
- this.cursor = cursor;
- }
-
-
- public Query withCursor( String cursor ) {
- setCursor( cursor );
- return this;
- }
-
-
- public int getLimit() {
- return getLimit( DEFAULT_LIMIT );
- }
-
-
- public int getLimit( int defaultLimit ) {
- if ( limit <= 0 ) {
- if ( defaultLimit > 0 ) {
- return defaultLimit;
- }
- else {
- return DEFAULT_LIMIT;
- }
- }
- return limit;
- }
-
-
- public void setLimit( int limit ) {
-
- // TODO tnine. After users have had time to change their query limits,
- // this needs to be uncommented and enforced.
- // if(limit > MAX_LIMIT){
- // throw new IllegalArgumentException(
- // String.format("Query limit must be <= to %d", MAX_LIMIT));
- // }
-
- if ( limit > MAX_LIMIT ) {
- limit = MAX_LIMIT;
- }
-
- this.limit = limit;
- }
-
-
- public Query withLimit( int limit ) {
- setLimit( limit );
- return this;
- }
public boolean isReversed() {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
index 6df5a6d..2c03635 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/index/impl/EntityIndexTest.java
@@ -118,7 +118,7 @@ public class EntityIndexTest extends BaseIT {
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
EntityIndexBatch batch = entityIndex.createBatch();
Entity entity = new Entity( entityType );
EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID());
@@ -143,7 +143,7 @@ public class EntityIndexTest extends BaseIT {
ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
}
@Test
@@ -219,17 +219,17 @@ public class EntityIndexTest extends BaseIT {
final String entityType = "thing";
IndexScope indexScope = new IndexScopeImpl( appId, "things" );
- final SearchTypes searchTypes = SearchTypes.fromTypes( entityType );
+ final SearchTypes searchTypes = SearchTypes.fromTypes(entityType);
insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,0);
ei.refresh();
- testQueries( indexScope, searchTypes, entityIndex );
+ testQueries(indexScope, searchTypes, entityIndex);
- ei.addIndex("v2", 1,0,"one");
+ ei.addIndex("v2", 1, 0, "one");
- insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json",101,100);
+ insertJsonBlob(entityIndex, entityType, indexScope, "/sample-large.json", 101, 100);
ei.refresh();
@@ -364,11 +364,12 @@ public class EntityIndexTest extends BaseIT {
StopWatch timer = new StopWatch();
timer.start();
Query query = Query.fromQL( queryString );
- query.setLimit( 1000 );
- CandidateResults candidateResults = entityIndex.search( scope, searchTypes, query );
+ CandidateResults candidateResults = null;
+ candidateResults = entityIndex.search( scope, searchTypes, query,num+1 );
+
timer.stop();
- assertEquals( num, candidateResults.size() );
+ assertEquals( num,candidateResults.size() );
log.debug( "Query time {}ms", timer.getTime() );
return candidateResults;
}
@@ -652,13 +653,8 @@ public class EntityIndexTest extends BaseIT {
for ( int i = 0; i < expectedPages; i++ ) {
//**
Query query = Query.fromQL( "select * order by created" );
- query.setLimit( limit );
-
- if ( cursor != null ) {
- query.setCursor( cursor );
- }
- final CandidateResults results = entityIndex.search( indexScope, SearchTypes.allTypes(), query );
+ final CandidateResults results = cursor == null ? entityIndex.search( indexScope, SearchTypes.allTypes(), query , limit) : entityIndex.getNextPage(cursor);
assertTrue( results.hasCursor() );
@@ -667,8 +663,6 @@ public class EntityIndexTest extends BaseIT {
assertEquals("Should be 16 bytes as hex", 32, cursor.length());
-
-
assertEquals( 1, results.size() );
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/8a62d117/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index 102e222..792fe3a 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -19,6 +19,9 @@
package org.apache.usergrid.persistence.query.tree;
+import org.apache.usergrid.persistence.core.scope.ApplicationScopeImpl;
+import org.apache.usergrid.persistence.index.impl.IndexScopeImpl;
+import org.apache.usergrid.persistence.index.impl.SearchRequestBuilderStrategy;
import org.apache.usergrid.persistence.index.query.tree.LongLiteral;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterLexer;
import org.apache.usergrid.persistence.index.query.tree.CpQueryFilterParser;
@@ -43,6 +46,7 @@ import org.antlr.runtime.ANTLRStringStream;
import org.antlr.runtime.RecognitionException;
import org.antlr.runtime.TokenRewriteStream;
import org.apache.usergrid.persistence.index.exceptions.QueryParseException;
+import org.apache.usergrid.persistence.model.entity.SimpleId;
import org.junit.Test;
import org.apache.usergrid.persistence.index.query.Query;
import org.elasticsearch.index.query.QueryBuilder;
@@ -429,7 +433,9 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- QueryBuilder qb = query.createQueryBuilder(new String[]{"testcontext"});
+ SearchRequestBuilderStrategy builderStrategy = new SearchRequestBuilderStrategy(null,new ApplicationScopeImpl(new SimpleId("test")),null,100);
+ QueryBuilder qb =builderStrategy.createQueryBuilder(new IndexScopeImpl(new SimpleId("owner"),"app"),query);
+
}