You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by gr...@apache.org on 2015/03/18 00:28:56 UTC
incubator-usergrid git commit: [USERGRID-460] Added ability to make
geo sorts possible. Not Working. Refacorted the query code for filterBuilders
and queryBuilders in the Query class.
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-460 65ff858db -> 65c5e22d1
[USERGRID-460] Added ability to make geo sorts possible. Not Working.
Refacorted the query code for filterBuilders and queryBuilders in the Query class.
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/65c5e22d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/65c5e22d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/65c5e22d
Branch: refs/heads/USERGRID-460
Commit: 65c5e22d18d7eaba6e41ff7d84eb51365da6930d
Parents: 65ff858
Author: GERey <gr...@apigee.com>
Authored: Tue Mar 17 16:28:55 2015 -0700
Committer: GERey <gr...@apigee.com>
Committed: Tue Mar 17 16:28:55 2015 -0700
----------------------------------------------------------------------
.../cassandra/QueryProcessorImpl.java | 15 +++-
.../index/impl/EsEntityIndexImpl.java | 77 +++++++++++++-------
.../persistence/index/impl/EsQueryVistor.java | 61 +++++++++-------
.../usergrid/persistence/index/query/Query.java | 36 ++-------
.../index/query/tree/QueryVisitor.java | 5 +-
5 files changed, 107 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
----------------------------------------------------------------------
diff --git a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
index 874ff88..7c4ea37 100644
--- a/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
+++ b/stack/core/src/main/java/org/apache/usergrid/persistence/cassandra/QueryProcessorImpl.java
@@ -71,6 +71,7 @@ import org.apache.usergrid.persistence.query.ir.result.ScanColumn;
import org.apache.usergrid.persistence.schema.CollectionInfo;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -292,7 +293,7 @@ public class QueryProcessorImpl implements QueryProcessor {
}
}
if (logger.isDebugEnabled()) {
- logger.debug("Getting result for query: [{}], returning entityIds size: {}",
+ logger.debug("Getting result for query: [{}], returning entityIds size: {}",
getQuery(), entityIds.size());
}
@@ -621,12 +622,18 @@ public class QueryProcessorImpl implements QueryProcessor {
@Override
public QueryBuilder getQueryBuilder() {
- throw new UnsupportedOperationException("Not supported by this vistor implementation.");
+ throw new UnsupportedOperationException("Not supported by this vistor implementation.");
}
@Override
public FilterBuilder getFilterBuilder() {
- throw new UnsupportedOperationException("Not supported by this vistor implementation.");
+ throw new UnsupportedOperationException("Not supported by this vistor implementation.");
+ }
+
+
+ @Override
+ public GeoDistanceSortBuilder getGeoDistanceSortBuilder() {
+ throw new UnsupportedOperationException("Not supported by this vistor implementation.");
}
}
@@ -724,4 +731,4 @@ public class QueryProcessorImpl implements QueryProcessor {
public EntityManager getEntityManager() {
return em;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index c92b299..ed1f03a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -39,6 +39,7 @@ import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.query.CandidateResult;
import org.apache.usergrid.persistence.index.query.CandidateResults;
import org.apache.usergrid.persistence.index.query.Query;
+import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.UUIDUtils;
import org.apache.usergrid.persistence.map.MapManager;
import org.apache.usergrid.persistence.map.MapManagerFactory;
@@ -64,11 +65,14 @@ import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequestBuilder;
import org.elasticsearch.client.AdminClient;
+import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.DistanceUnit;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.index.query.*;
+import org.elasticsearch.index.search.geo.GeoDistanceFilter;
import org.elasticsearch.indices.IndexAlreadyExistsException;
import org.elasticsearch.indices.IndexMissingException;
import org.elasticsearch.indices.InvalidAliasNameException;
@@ -397,6 +401,14 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
}
+ /**
+ * Needs REfactor to make clearer what queries and how the information gets used so we don't need to retraverse
+ * the query every time we generate some information about it.
+ * @param indexScope
+ * @param searchTypes
+ * @param query
+ * @return
+ */
@Override
public CandidateResults search( final IndexScope indexScope, final SearchTypes searchTypes,
final Query query ) {
@@ -404,8 +416,9 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
final String context = IndexingUtils.createContextName(indexScope);
final String[] entityTypes = searchTypes.getTypeNames();
- QueryBuilder qb = query.createQueryBuilder(context);
+ final QueryVisitor queryVisitor = query.getQueryVisitor();
+ QueryBuilder qb = query.createQueryBuilder( context );
SearchResponse searchResponse;
@@ -415,8 +428,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
.setScroll(cursorTimeout + "m")
.setQuery(qb);
- final FilterBuilder fb = query.createFilterBuilder();
-
+ final FilterBuilder fb = queryVisitor.getFilterBuilder();
//we have post filters, apply them
if ( fb != null ) {
logger.debug( " Filter: {} ", fb.toString() );
@@ -426,6 +438,7 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
srb = srb.setFrom( 0 ).setSize( query.getLimit() );
+
for ( Query.SortPredicate sp : query.getSortPredicates() ) {
final SortOrder order;
@@ -436,34 +449,44 @@ public class EsEntityIndexImpl implements AliasedEntityIndex {
order = SortOrder.DESC;
}
- // we do not know the type of the "order by" property and so we do not know what
- // type prefix to use. So, here we add an order by clause for every possible type
- // that you can order by: string, number and boolean and we ask ElasticSearch
- // to ignore any fields that are not present.
-
- final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
- final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( stringSort );
-
- logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
-
- final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
- final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( numberSort );
- logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
-
- final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
- final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName )
- .order( order ).ignoreUnmapped( true );
- srb.addSort( booleanSort );
- logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
+
+ //By default we want to order geo location queries to be returned by the order defined above first.
+ /**
+ * I'm not sure there is any way to build the sort without traversing the tree.
+ */
+ if(fb instanceof GeoDistanceFilterBuilder){
+ srb.addSort(queryVisitor.getGeoDistanceSortBuilder().order( SortOrder.ASC ).unit( DistanceUnit.KILOMETERS ).geoDistance(
+ GeoDistance.SLOPPY_ARC ));
+ logger.info( " Geo Sort: {} order by {}", sp.getPropertyName(), order.toString() );
+
+ }
+ else {
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
+ final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder stringSort = SortBuilders.fieldSort( stringFieldName ).order( order ).ignoreUnmapped( true );
+ srb.addSort( stringSort );
+
+ logger.debug( " Sort: {} order by {}", stringFieldName, order.toString() );
+
+ final String numberFieldName = NUMBER_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder numberSort = SortBuilders.fieldSort( numberFieldName ).order( order ).ignoreUnmapped( true );
+ srb.addSort( numberSort );
+ logger.debug( " Sort: {} order by {}", numberFieldName, order.toString() );
+
+ final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
+ final FieldSortBuilder booleanSort = SortBuilders.fieldSort( booleanFieldName ).order( order )
+ .ignoreUnmapped( true );
+ srb.addSort( booleanSort );
+ logger.debug( " Sort: {} order by {}", booleanFieldName, order.toString() );
+ }
}
if ( logger.isDebugEnabled() ) {
- logger.debug( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
+ logger.info( "Searching index (read alias): {}\n scope: {} \n type: {}\n query: {} ",
this.alias.getReadAlias(), context, entityTypes, srb );
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index f012bab..2e7b174 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -30,6 +30,8 @@ import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
+import org.elasticsearch.search.sort.SortBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -58,7 +60,7 @@ import static org.apache.usergrid.persistence.index.impl.IndexingUtils.STRING_PR
/**
- * Visits tree of parsed Query operands and populates
+ * Visits tree of parsed Query operands and populates
* ElasticSearch QueryBuilder that represents the query.
*/
public class EsQueryVistor implements QueryVisitor {
@@ -66,8 +68,9 @@ public class EsQueryVistor implements QueryVisitor {
Stack<QueryBuilder> stack = new Stack<QueryBuilder>();
List<FilterBuilder> filterBuilders = new ArrayList<FilterBuilder>();
+ GeoDistanceSortBuilder geoSortBuilder = null;
+
-
@Override
public void visit( AndOperand op ) throws IndexException {
@@ -157,14 +160,14 @@ public class EsQueryVistor implements QueryVisitor {
Object value = op.getLiteral().getValue();
BoolQueryBuilder qb = QueryBuilders.boolQuery(); // let's do a boolean OR
- qb.minimumNumberShouldMatch(1);
+ qb.minimumNumberShouldMatch(1);
// field is an entity/array that needs no name prefix
qb = qb.should( QueryBuilders.matchQuery( name, value ) );
// OR field is a string and needs the prefix on the name
qb = qb.should( QueryBuilders.matchQuery( addPrefix( value.toString(), name, true), value));
-
+
stack.push( qb );
}
@@ -186,7 +189,8 @@ public class EsQueryVistor implements QueryVisitor {
FilterBuilder fb = FilterBuilders.geoDistanceFilter( name )
.lat( lat ).lon( lon ).distance( distance, DistanceUnit.METERS );
filterBuilders.add( fb );
- }
+ geoSortBuilder = SortBuilders.geoDistanceSort( "location" ).point( lat,lon );
+ }
@Override
@@ -222,19 +226,19 @@ public class EsQueryVistor implements QueryVisitor {
qb.minimumNumberShouldMatch(1);
// field is an entity/array that does not need a prefix on its name
- // TODO is this right now that we've updated our doc structure?
+ // TODO is this right now that we've updated our doc structure?
// Should this be "must" instead of should?
qb = qb.should( QueryBuilders.wildcardQuery( name, svalue ) );
-
+
// or field is just a string that does need a prefix
if ( svalue.indexOf("*") != -1 ) {
qb = qb.should( QueryBuilders.wildcardQuery( addPrefix( value, name ), svalue ) );
} else {
qb = qb.should( QueryBuilders.termQuery( addPrefix( value, name ), value ));
- }
+ }
stack.push( qb );
return;
- }
+ }
// assume all other types need prefix
stack.push( QueryBuilders.termQuery( addPrefix( value, name ), value ));
@@ -276,7 +280,7 @@ public class EsQueryVistor implements QueryVisitor {
if ( parts.length > 1 ) {
name = parts[ parts.length - 1 ];
}
-
+
if ( value instanceof String && analyzed ) {
name = addAnalyzedStringPrefix( name );
@@ -293,12 +297,12 @@ public class EsQueryVistor implements QueryVisitor {
name = addStringPrefix( name );
}
- // re-create nested property name
+ // re-create nested property name
if ( parts.length > 1 ) {
parts[parts.length - 1] = name;
Joiner joiner = Joiner.on(".").skipNulls();
return joiner.join(parts);
- }
+ }
return name;
}
@@ -307,34 +311,34 @@ public class EsQueryVistor implements QueryVisitor {
private String addAnalyzedStringPrefix( String name ) {
if ( name.startsWith( ANALYZED_STRING_PREFIX ) ) {
return name;
- }
+ }
return ANALYZED_STRING_PREFIX + name;
- }
-
+ }
+
private String addStringPrefix( String name ) {
if ( name.startsWith( STRING_PREFIX ) ) {
return name;
- }
+ }
return STRING_PREFIX + name;
- }
-
+ }
+
private String addNumberPrefix( String name ) {
if ( name.startsWith( NUMBER_PREFIX ) ) {
return name;
- }
+ }
return NUMBER_PREFIX + name;
- }
-
+ }
+
private String addBooleanPrefix( String name ) {
if ( name.startsWith( BOOLEAN_PREFIX ) ) {
return name;
- }
+ }
return BOOLEAN_PREFIX + name;
- }
-
+ }
+
@Override
public QueryBuilder getQueryBuilder() {
@@ -354,14 +358,19 @@ public class EsQueryVistor implements QueryVisitor {
for ( FilterBuilder fb : filterBuilders ) {
if ( andFilter == null ) {
andFilter = FilterBuilders.andFilter( fb );
- } else {
+ } else {
andFilter = FilterBuilders.andFilter( andFilter, fb );
}
- }
+ }
} else if ( !filterBuilders.isEmpty() ) {
return filterBuilders.get(0);
}
return null;
}
+
+ @Override
+ public GeoDistanceSortBuilder getGeoDistanceSortBuilder(){
+ return geoSortBuilder;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
index da68772..ed6d19e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/Query.java
@@ -103,6 +103,7 @@ public class Query {
private List<CounterFilterPredicate> counterFilters;
private String collection;
private String ql;
+ private QueryVisitor visitor;
private static ObjectMapper mapper = new ObjectMapper();
@@ -146,27 +147,7 @@ public class Query {
public QueryBuilder createQueryBuilder( final String context ) {
- QueryBuilder queryBuilder = null;
-
-
- //we have a root operand. Translate our AST into an ES search
- if ( getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
-
- try {
- getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
-
-
- queryBuilder = v.getQueryBuilder();
- }
-
+ QueryBuilder queryBuilder = getQueryVisitor().getQueryBuilder();
// Add our filter for context to our query for fast execution.
// Fast because it utilizes bitsets internally. See this post for more detail.
@@ -188,23 +169,20 @@ public class Query {
}
- public FilterBuilder createFilterBuilder() {
- FilterBuilder filterBuilder = null;
+ public QueryVisitor getQueryVisitor() {
+
+ QueryVisitor v = new EsQueryVistor();
if ( getRootOperand() != null ) {
- QueryVisitor v = new EsQueryVistor();
try {
getRootOperand().visit( v );
} catch ( IndexException ex ) {
throw new RuntimeException( "Error building ElasticSearch query", ex );
}
- filterBuilder = v.getFilterBuilder();
}
-
- return filterBuilder;
- }
-
+ return v;
+ }
/**
* Create a query instance from the QL. If the string is null, return an empty query
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/65c5e22d/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
index d295c92..943718c 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
@@ -24,6 +24,7 @@ import org.apache.usergrid.persistence.index.exceptions.NoIndexException;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.elasticsearch.index.query.FilterBuilder;
import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
/**
@@ -93,10 +94,12 @@ public interface QueryVisitor {
*/
public void visit( GreaterThanEqual op ) throws NoIndexException;
- /**
+ /**
* Returns resulting query builder.
*/
public QueryBuilder getQueryBuilder();
public FilterBuilder getFilterBuilder();
+
+ public GeoDistanceSortBuilder getGeoDistanceSortBuilder();
}