You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/06 19:40:16 UTC
[2/2] incubator-usergrid git commit: Upgraded to lastest ES version.
Made node client the default. Can be configured with
elasticsearch.client.type
Upgraded to lastest ES version. Made node client the default. Can be configured with elasticsearch.client.type
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/5a40ff55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/5a40ff55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/5a40ff55
Branch: refs/heads/USERGRID-536
Commit: 5a40ff553bb42805c2abb5a9ba067c9b9c8f8c63
Parents: 543e13b
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 6 11:40:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 6 11:40:12 2015 -0600
----------------------------------------------------------------------
stack/corepersistence/pom.xml | 2 +-
.../usergrid/persistence/index/IndexFig.java | 18 ++
.../impl/EsApplicationEntityIndexImpl.java | 12 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 25 +--
.../persistence/index/impl/EsProvider.java | 129 +++++++++++--
.../persistence/index/impl/IndexingUtils.java | 17 +-
.../impl/SearchRequestBuilderStrategy.java | 179 ++++++++++---------
.../persistence/index/query/SortPredicate.java | 20 ++-
.../index/query/tree/QueryVisitor.java | 25 +--
.../persistence/query/tree/GrammarTreeTest.java | 6 +-
10 files changed, 289 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index f26f1cc..8494996 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -59,7 +59,7 @@ limitations under the License.
<commons.collections.version>3.2.1</commons.collections.version>
<commons.io.version>2.4</commons.io.version>
<commons.lang.version>3.1</commons.lang.version>
- <elasticsearch.version>1.3.2</elasticsearch.version>
+ <elasticsearch.version>1.4.4</elasticsearch.version>
<fasterxml-uuid.version>3.1.3</fasterxml-uuid.version>
<guava.version>18.0</guava.version>
<guice.version>4.0-beta5</guice.version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index c7be79d..712c35d 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -24,6 +24,8 @@ import org.safehaus.guicyfig.FigSingleton;
import org.safehaus.guicyfig.GuicyFig;
import org.safehaus.guicyfig.Key;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+
@FigSingleton
public interface IndexFig extends GuicyFig {
@@ -100,6 +102,14 @@ public interface IndexFig extends GuicyFig {
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+
+
+ /**
+ * The queue implementation to use. Values come from <class>QueueProvider.Implementations</class>
+ */
+ public static final String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
+
+
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
String getHosts();
@@ -209,4 +219,12 @@ public interface IndexFig extends GuicyFig {
@Default( "1000" )
@Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
long getQueueOfferTimeout();
+
+ /**
+ * Return the type of client. Valid values or NODE or TRANSPORT
+ * @return
+ */
+ @Key( ELASTICSEARCH_CLIENT_TYPE )
+ @Default( "NODE")
+ String getClientType();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index d76dab7..9db1c26 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.index.impl;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -90,7 +91,6 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final MapManager mapManager;
private final AliasedEntityIndex entityIndex;
private final IndexBufferProducer indexBatchBufferProducer;
- private final IndexCache indexCache;
private final IndexFig indexFig;
private final EsProvider esProvider;
private final IndexAlias alias;
@@ -102,15 +102,14 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
@Inject
- public EsApplicationEntityIndexImpl( @Assisted ApplicationScope appScope, final AliasedEntityIndex entityIndex,
+ public EsApplicationEntityIndexImpl( ApplicationScope appScope, final AliasedEntityIndex entityIndex,
final IndexFig config, final IndexBufferProducer indexBatchBufferProducer,
- final EsProvider provider, final IndexCache indexCache,
+ final EsProvider provider,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig,
final FailureMonitorImpl.IndexIdentifier indexIdentifier ) {
this.entityIndex = entityIndex;
this.indexBatchBufferProducer = indexBatchBufferProducer;
- this.indexCache = indexCache;
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
ValidationUtils.validateApplicationScope( appScope );
@@ -314,8 +313,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings() );
final String esScrollCursor = searchResponse.getScrollId();
- // >= seems odd. However if our user reduces expectedSize (limit) on subsequent requests, we can't do that
- //therefor we need to account for the overflow
+ // >= seems odd. However if we get an overflow, we need to account for it.
if ( esScrollCursor != null && length >= limit ) {
final String cursor = candidateResults.initializeCursor();
@@ -340,7 +338,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
/**
* Class to encapsulate our serialized state
*/
- private static final class QueryState {
+ private static final class QueryState implements Serializable{
/**
* Our reserved character for constructing our storage string
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 648a29d..6145069 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -19,18 +19,21 @@
*/
package org.apache.usergrid.persistence.index.impl;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
+import java.util.concurrent.ExecutionException;
+
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.map.MapManagerFactory;
-import java.util.concurrent.ExecutionException;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
/**
* Get index from factory, adds caching
@@ -39,7 +42,6 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
- private final IndexCache indexCache;
private final IndexBufferProducer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
private final MapManagerFactory mapManagerFactory;
@@ -51,19 +53,18 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, ApplicationEntityIndex>() {
public ApplicationEntityIndex load( ApplicationScope scope ) {
return new EsApplicationEntityIndexImpl(
- scope,entityIndex,config, indexBatchBufferProducer, provider,indexCache, metricsFactory, mapManagerFactory, indexFig, indexIdentifier
+ scope,entityIndex,config, indexBatchBufferProducer, provider, metricsFactory, mapManagerFactory, indexFig, indexIdentifier
);
}
} );
@Inject
- public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final IndexCache indexCache,
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider,
final IndexBufferProducer indexBatchBufferProducer,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig, final AliasedEntityIndex entityIndex, final FailureMonitorImpl.IndexIdentifier indexIdentifier ){
this.config = config;
this.provider = provider;
- this.indexCache = indexCache;
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.metricsFactory = metricsFactory;
this.mapManagerFactory = mapManagerFactory;
@@ -75,7 +76,7 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
@Override
- public ApplicationEntityIndex createApplicationEntityIndex(@Assisted final ApplicationScope appScope) {
+ public ApplicationEntityIndex createApplicationEntityIndex(final ApplicationScope appScope) {
try{
return eiCache.get(appScope);
}catch (ExecutionException ee){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index 2ea6774..096bd17 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -25,7 +25,10 @@ import java.net.UnknownHostException;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +53,7 @@ public class EsProvider {
public static String LOCAL_ES_PORT_PROPNAME = "EMBEDDED_ES_PORT";
+
@Inject
public EsProvider( IndexFig fig ) {
this.indexFig = fig;
@@ -62,37 +66,67 @@ public class EsProvider {
public Client getClient() {
if ( client == null ) {
//synchronize on creating the client so we don't create too many
- createClient( indexFig );
+ createClient();
}
return client;
}
/**
- * Reset the client instnace
+ * Reset the client instance
*/
- public void releaseClient() {
+ public synchronized void releaseClient() {
//reset our static variables
if ( client != null ) {
- client = null;
+ try {
+ client.close();
+ }
+ //if we fail for any reason, null it so the next request creates a new client
+ finally {
+ client = null;
+ }
}
}
- private synchronized void createClient( IndexFig fig ) {
+ /**
+ * Create our client
+ */
+ private synchronized void createClient() {
- if ( client != null) {
+ if ( client != null ) {
return;
}
- final String clusterName = fig.getClusterName();
- final int port = fig.getPort();
- ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
- .put( "cluster.name", clusterName )
- .put( "client.transport.sniff", true );
+ final ClientType clientType = ClientType.valueOf( indexFig.getClientType() );
+
+ switch ( clientType ) {
+ case NODE:
+ client = createNodeClient();
+ break;
+
+ case TRANSPORT:
+ client = createTransportClient();
+ break;
+ default:
+ throw new RuntimeException( "Only client types of NODE and TRANSPORT are supported" );
+ }
+ }
+
+
+ /**
+ * Create the transport client
+ * @return
+ */
+ private Client createTransportClient() {
+ final String clusterName = indexFig.getClusterName();
+ final int port = indexFig.getPort();
+
+ ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName )
+ .put( "client.transport.sniff", true );
- String nodeName = fig.getNodeName();
+ String nodeName = indexFig.getNodeName();
if ( "default".equals( nodeName ) ) {
// no nodeName was specified, use hostname
@@ -105,17 +139,76 @@ public class EsProvider {
}
}
- settings.put( "node.name", nodeName);
+ settings.put( "node.name", nodeName );
+
TransportClient transportClient = new TransportClient( settings.build() );
- // we will connect to ES on all configured hosts
- for ( String host : fig.getHosts().split( "," ) ) {
- transportClient.addTransportAddress( new InetSocketTransportAddress(host, port));
- }
- client = transportClient;
+ // we will connect to ES on all configured hosts
+ for ( String host : indexFig.getHosts().split( "," ) ) {
+ transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) );
+ }
+
+ return transportClient;
}
+ /**
+ * Create a node client
+ * @return
+ */
+ public Client createNodeClient() {
+
+ // we will connect to ES on all configured hosts
+
+
+ final String clusterName = indexFig.getClusterName();
+ final String nodeName = indexFig.getNodeName();
+ final int port = indexFig.getPort();
+
+ /**
+ * Create our hosts
+ */
+ final StringBuffer hosts = new StringBuffer();
+
+ for ( String host : indexFig.getHosts().split( "," ) ) {
+ hosts.append( host ).append( ":" ).append( port ).append( "," );
+ }
+
+ //remove the last comma
+ hosts.deleteCharAt( hosts.length() - 1 );
+
+ final String hostString = hosts.toString();
+
+
+ Settings settings = ImmutableSettings.settingsBuilder()
+
+ .put( "cluster.name", clusterName )
+ // this assumes that we're using zen for host discovery. Putting an
+ // explicit set of bootstrap hosts ensures we connect to a valid cluster.
+ .put( "discovery.zen.ping.unicast.hosts", hostString )
+ .put( "discovery.zen.ping.multicast.enabled", "false" ).put( "http.enabled", false )
+
+ .put( "client.transport.ping_timeout", 2000 ) // milliseconds
+ .put( "client.transport.nodes_sampler_interval", 100 ).put( "network.tcp.blocking", true )
+ .put( "node.client", true ).put( "node.name", nodeName )
+
+ .build();
+
+ log.debug( "Creating ElasticSearch client with settings: {}", settings.getAsMap() );
+
+ Node node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).data( false ).node();
+
+ return node.client();
+ }
+
+
+ /**
+ *
+ */
+ public enum ClientType {
+ TRANSPORT,
+ NODE
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 0b88dc5..299b95b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -44,7 +44,7 @@ public class IndexingUtils {
public static final String ES_ENTITY_TYPE = "entity";
/**
- * Reserved UG fields in the document
+ * Reserved UG fields in the document
*/
public static final String APPLICATION_ID_FIELDNAME = "applicationId";
@@ -79,6 +79,19 @@ public class IndexingUtils {
public static final String FIELD_LOCATION = "location";
public static final String FIELD_STRING = "string";
+
+ /**
+ * All possible sort values
+ */
+ public static final String SORT_FIELD_BOOLEAN = ENTITY_FIELDS + ".boolean";
+ public static final String SORT_FIELD_INT = ENTITY_FIELDS + ".int";
+ public static final String SORT_FIELD_LONG = ENTITY_FIELDS + ".long";
+ public static final String SORT_FIELD_FLOAT = ENTITY_FIELDS + ".float";
+ public static final String SORT_FIELD_DOUBLE = ENTITY_FIELDS + ".double";
+ public static final String SORT_FIELD_LOCATION = ENTITY_FIELDS + ".location";
+ public static final String SORT_FIELD_STRING = ENTITY_FIELDS + ".string";
+
+
//The value appended to the string field when it's an exact match. Should only be used on search, never index
public static final String FIELD_STRING_EQUALS = FIELD_STRING + ".exact";
@@ -154,8 +167,6 @@ public class IndexingUtils {
/**
* Parse the document id into a candidate result
- * @param documentId
- * @return
*/
public static CandidateResult parseIndexDocId( final String documentId ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 18d4228..1e0ecde 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -23,8 +23,10 @@ package org.apache.usergrid.persistence.index.impl;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermFilterBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
@@ -44,20 +46,23 @@ import com.google.common.base.Preconditions;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+
/**
* The strategy for creating a search request from a parsed query
*/
public class SearchRequestBuilderStrategy {
- private static final Logger logger = LoggerFactory.getLogger(SearchRequestBuilderStrategy.class);
+ private static final Logger logger = LoggerFactory.getLogger( SearchRequestBuilderStrategy.class );
private final EsProvider esProvider;
private final ApplicationScope applicationScope;
private final IndexAlias alias;
private final int cursorTimeout;
- public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){
+
+ public SearchRequestBuilderStrategy( final EsProvider esProvider, final ApplicationScope applicationScope,
+ final IndexAlias alias, int cursorTimeout ) {
this.esProvider = esProvider;
this.applicationScope = applicationScope;
@@ -65,94 +70,75 @@ public class SearchRequestBuilderStrategy {
this.cursorTimeout = cursorTimeout;
}
- public SearchRequestBuilder getBuilder(final SearchEdge searchEdge, final SearchTypes searchTypes, final ParsedQuery query, final int limit) {
- Preconditions.checkArgument(limit <= EntityIndex.MAX_LIMIT, "limit is greater than max "+ EntityIndex.MAX_LIMIT);
+ public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes searchTypes,
+ final ParsedQuery query, final int limit ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
- .setTypes(searchTypes.getTypeNames(applicationScope))
- .setScroll(cursorTimeout + "m")
- .setQuery(createQueryBuilder( searchEdge,query));
+ Preconditions
+ .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max " + EntityIndex.MAX_LIMIT );
- final FilterBuilder fb = createFilterBuilder(query);
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
+ .setTypes( searchTypes.getTypeNames( applicationScope ) )
+ .setScroll( cursorTimeout + "m" );
- //we have post filters, apply them
- if (fb != null) {
- logger.debug(" Filter: {} ", fb.toString());
- srb = srb.setPostFilter(fb);
- }
+ final QueryVisitor visitor = visitParsedQuery(query);
+
+ srb.setQuery( createQueryBuilder( searchEdge, visitor ) );
+
+ final FilterBuilder fb = visitor.getFilterBuilder();
+
+ srb.setPostFilter( fb );
- srb = srb.setFrom(0).setSize(limit);
-
- for (SortPredicate sp : query.getSortPredicates()) {
- throw new RuntimeException( "Fix me" );
-
-// final SortOrder order;
-// if (sp.getDirection().equals( SortPredicate.SortDirection.ASCENDING)) {
-// order = SortOrder.ASC;
-// } else {
-// order = SortOrder.DESC;
-// }
-//
-// // we do not know the type of the "order by" property and so we do not know what
-// // type prefix to use. So, here we add an order by clause for every possible type
-// // that you can order by: string, number and boolean and we ask ElasticSearch
-// // to ignore any fields that are not present.
-//
-// final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(stringSort);
-//
-// logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
-//
-// final String longFieldName = LONG_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(longSort);
-// logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
-//
-//
-// final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(doubleSort);
-// logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
-//
-//
-// final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(booleanSort);
-// logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+
+ srb = srb.setFrom( 0 ).setSize( limit );
+
+ //no sort predicates, sort by edge time descending, entity id second
+ if ( query.getSortPredicates().size() == 0 ) {
+ //sort by the edge timestamp
+ srb.addSort( SortBuilders.fieldSort( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME ).order( SortOrder.DESC ) );
+
+ //sort by the entity id if our times are equal
+ srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) );
}
- return srb;
- }
+ //we have sort predicates, sort them
+ for ( SortPredicate sp : query.getSortPredicates() ) {
- public QueryBuilder createQueryBuilder(final SearchEdge searchEdge, final ParsedQuery query) {
- String context = createContextName(applicationScope, searchEdge );
- QueryBuilder queryBuilder = null;
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
- //we have a root operand. Translate our AST into an ES search
- if ( query.getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
+ final SortOrder order = sp.getDirection().toEsSort();
+ final String propertyName = sp.getPropertyName();
- try {
- query.getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_STRING, propertyName ) );
+
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_INT, propertyName ) );
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_DOUBLE, propertyName ) );
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_BOOLEAN, propertyName ) );
+
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_LONG, propertyName ) );
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_FLOAT, propertyName ) );
- queryBuilder = v.getQueryBuilder();
}
+ return srb;
+ }
+
+
+ public QueryBuilder createQueryBuilder( final SearchEdge searchEdge, final QueryVisitor visitor ) {
+ String context = createContextName( applicationScope, searchEdge );
+
+ QueryBuilder queryBuilder = visitor.getQueryBuilder();
// Add our filter for context to our query for fast execution.
@@ -165,7 +151,7 @@ public class SearchRequestBuilderStrategy {
//make sure we have entity in the context
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
- boolQueryBuilder.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
+ boolQueryBuilder.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
boolQueryBuilder.must( queryBuilder );
@@ -173,21 +159,44 @@ public class SearchRequestBuilderStrategy {
}
- public FilterBuilder createFilterBuilder(ParsedQuery query) {
- FilterBuilder filterBuilder = null;
- if ( query.getRootOperand() != null ) {
- QueryVisitor v = new EsQueryVistor();
- try {
- query.getRootOperand().visit( v );
- } catch ( IndexException ex ) {
+ /**
+ * Perform our visit of the query once for efficiency
+ * @param parsedQuery
+ * @return
+ */
+ private QueryVisitor visitParsedQuery(final ParsedQuery parsedQuery){
+ QueryVisitor v = new EsQueryVistor();
+
+ if ( parsedQuery.getRootOperand() != null ) {
+
+ try {
+ parsedQuery.getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
throw new RuntimeException( "Error building ElasticSearch query", ex );
}
- filterBuilder = v.getFilterBuilder();
+
}
- return filterBuilder;
+ return v;
}
+
+ /**
+ * Create a sort for the property name and field name specified
+ * @param sortOrder The sort order
+ * @param fieldName The name of the field for the type
+ * @param propertyName The property name the user specified for the sort
+ * @return
+ */
+ private FieldSortBuilder createSort( final SortOrder sortOrder, final String fieldName,
+ final String propertyName ) {
+
+ final TermFilterBuilder propertyFilter = FilterBuilders.termFilter( IndexingUtils.FIELD_NAME, propertyName );
+
+
+ return SortBuilders.fieldSort( fieldName ).order( sortOrder ).ignoreUnmapped( true ).setNestedFilter( propertyFilter );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
index 70d0b30..92636fb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
@@ -26,6 +26,8 @@ package org.apache.usergrid.persistence.index.query;
import java.io.Serializable;
+import org.elasticsearch.search.sort.SortOrder;
+
/**
* An object that represents a sort predicate
@@ -100,7 +102,23 @@ public final class SortPredicate implements Serializable {
public enum SortDirection {
- ASCENDING, DESCENDING;
+ ASCENDING(SortOrder.ASC), DESCENDING(SortOrder.DESC);
+
+ private final SortOrder esOrder;
+
+
+ SortDirection( final SortOrder esOrder ) {this.esOrder = esOrder;}
+
+
+ /**
+ * Get the ES sort direction
+ * @return
+ */
+ public SortOrder toEsSort(){
+ return esOrder;
+ }
+
+
public static SortDirection find( String s ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
index d295c92..5fc1304 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
@@ -38,65 +38,66 @@ public interface QueryVisitor {
* @param op
* @throws IndexException
*/
- public void visit( AndOperand op ) throws IndexException;
+ void visit( AndOperand op ) throws IndexException;
/**
* @param op
* @throws IndexException
*/
- public void visit( OrOperand op ) throws IndexException;
+ void visit( OrOperand op ) throws IndexException;
/**
* @param op
* @throws IndexException
*/
- public void visit( NotOperand op ) throws IndexException;
+ void visit( NotOperand op ) throws IndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( LessThan op ) throws NoIndexException;
+ void visit( LessThan op ) throws NoIndexException;
/**
* @param op
* @throws NoFullTextIndexException
*/
- public void visit( ContainsOperand op ) throws NoFullTextIndexException;
+ void visit( ContainsOperand op ) throws NoFullTextIndexException;
/**
* @param op
*/
- public void visit( WithinOperand op );
+ void visit( WithinOperand op );
/**
* @param op
* @throws NoIndexException
*/
- public void visit( LessThanEqual op ) throws NoIndexException;
+ void visit( LessThanEqual op ) throws NoIndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( Equal op ) throws NoIndexException;
+ void visit( Equal op ) throws NoIndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( GreaterThan op ) throws NoIndexException;
+ void visit( GreaterThan op ) throws NoIndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( GreaterThanEqual op ) throws NoIndexException;
+ void visit( GreaterThanEqual op ) throws NoIndexException;
/**
* Returns resulting query builder.
*/
- public QueryBuilder getQueryBuilder();
+ QueryBuilder getQueryBuilder();
- public FilterBuilder getFilterBuilder();
+
+ FilterBuilder getFilterBuilder();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/5a40ff55/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index ada7420..0ac0a59 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -436,11 +436,7 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- SearchRequestBuilderStrategy builderStrategy =
- new SearchRequestBuilderStrategy( null, new ApplicationScopeImpl( new SimpleId( "test" ) ), null, 100 );
- QueryBuilder qb =
- builderStrategy.createQueryBuilder( new SearchEdgeImpl( new SimpleId( "owner" ), "app",
- SearchEdge.NodeType.SOURCE ), query );
+
}