You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by to...@apache.org on 2015/04/06 19:40:45 UTC
incubator-usergrid git commit: Upgraded to latest ES version. Made
node client the default. Can be configured with elasticsearch.client.type
Repository: incubator-usergrid
Updated Branches:
refs/heads/USERGRID-536 5a40ff553 -> cb9b06a38 (forced update)
Upgraded to latest ES version. Made node client the default. Can be configured with elasticsearch.client.type
Project: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/commit/cb9b06a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/tree/cb9b06a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-usergrid/diff/cb9b06a3
Branch: refs/heads/USERGRID-536
Commit: cb9b06a38e77c0da931181d04784865af93c5ac6
Parents: 543e13b
Author: Todd Nine <tn...@apigee.com>
Authored: Mon Apr 6 11:40:12 2015 -0600
Committer: Todd Nine <tn...@apigee.com>
Committed: Mon Apr 6 11:40:38 2015 -0600
----------------------------------------------------------------------
stack/corepersistence/pom.xml | 2 +-
.../usergrid/persistence/index/IndexFig.java | 18 ++
.../impl/EsApplicationEntityIndexImpl.java | 12 +-
.../index/impl/EsEntityIndexFactoryImpl.java | 25 +--
.../persistence/index/impl/EsProvider.java | 129 +++++++++++--
.../persistence/index/impl/IndexingUtils.java | 17 +-
.../impl/SearchRequestBuilderStrategy.java | 179 ++++++++++---------
.../persistence/index/query/SortPredicate.java | 20 ++-
.../index/query/tree/QueryVisitor.java | 25 +--
.../persistence/query/tree/GrammarTreeTest.java | 6 +-
10 files changed, 289 insertions(+), 144 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index f26f1cc..8494996 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -59,7 +59,7 @@ limitations under the License.
<commons.collections.version>3.2.1</commons.collections.version>
<commons.io.version>2.4</commons.io.version>
<commons.lang.version>3.1</commons.lang.version>
- <elasticsearch.version>1.3.2</elasticsearch.version>
+ <elasticsearch.version>1.4.4</elasticsearch.version>
<fasterxml-uuid.version>3.1.3</fasterxml-uuid.version>
<guava.version>18.0</guava.version>
<guice.version>4.0-beta5</guice.version>
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
index c7be79d..0eebadc 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/IndexFig.java
@@ -24,6 +24,8 @@ import org.safehaus.guicyfig.FigSingleton;
import org.safehaus.guicyfig.GuicyFig;
import org.safehaus.guicyfig.Key;
+import org.apache.usergrid.persistence.index.impl.EsProvider;
+
@FigSingleton
public interface IndexFig extends GuicyFig {
@@ -100,6 +102,14 @@ public interface IndexFig extends GuicyFig {
public static final String QUERY_LIMIT_DEFAULT = "index.query.limit.default";
+
+
+ /**
+ * The client type to use. Valid values are NODE or TRANSPORT
+ */
+ public static final String ELASTICSEARCH_CLIENT_TYPE = "elasticsearch.client.type";
+
+
@Default( "127.0.0.1" )
@Key( ELASTICSEARCH_HOSTS )
String getHosts();
@@ -209,4 +219,12 @@ public interface IndexFig extends GuicyFig {
@Default( "1000" )
@Key( ELASTICSEARCH_QUEUE_OFFER_TIMEOUT )
long getQueueOfferTimeout();
+
+ /**
+ * Return the type of client. Valid values or NODE or TRANSPORT
+ * @return
+ */
+ @Key( ELASTICSEARCH_CLIENT_TYPE )
+ @Default( "NODE")
+ String getClientType();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
index d76dab7..9db1c26 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsApplicationEntityIndexImpl.java
@@ -20,6 +20,7 @@
package org.apache.usergrid.persistence.index.impl;
+import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
@@ -90,7 +91,6 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
private final MapManager mapManager;
private final AliasedEntityIndex entityIndex;
private final IndexBufferProducer indexBatchBufferProducer;
- private final IndexCache indexCache;
private final IndexFig indexFig;
private final EsProvider esProvider;
private final IndexAlias alias;
@@ -102,15 +102,14 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
@Inject
- public EsApplicationEntityIndexImpl( @Assisted ApplicationScope appScope, final AliasedEntityIndex entityIndex,
+ public EsApplicationEntityIndexImpl( ApplicationScope appScope, final AliasedEntityIndex entityIndex,
final IndexFig config, final IndexBufferProducer indexBatchBufferProducer,
- final EsProvider provider, final IndexCache indexCache,
+ final EsProvider provider,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig,
final FailureMonitorImpl.IndexIdentifier indexIdentifier ) {
this.entityIndex = entityIndex;
this.indexBatchBufferProducer = indexBatchBufferProducer;
- this.indexCache = indexCache;
this.indexFig = indexFig;
this.indexIdentifier = indexIdentifier;
ValidationUtils.validateApplicationScope( appScope );
@@ -314,8 +313,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
final CandidateResults candidateResults = new CandidateResults( candidates, query.getSelectFieldMappings() );
final String esScrollCursor = searchResponse.getScrollId();
- // >= seems odd. However if our user reduces expectedSize (limit) on subsequent requests, we can't do that
- //therefor we need to account for the overflow
+ // >= seems odd. However if we get an overflow, we need to account for it.
if ( esScrollCursor != null && length >= limit ) {
final String cursor = candidateResults.initializeCursor();
@@ -340,7 +338,7 @@ public class EsApplicationEntityIndexImpl implements ApplicationEntityIndex {
/**
* Class to encapsulate our serialized state
*/
- private static final class QueryState {
+ private static final class QueryState implements Serializable{
/**
* Our reserved character for constructing our storage string
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
index 648a29d..6145069 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexFactoryImpl.java
@@ -19,18 +19,21 @@
*/
package org.apache.usergrid.persistence.index.impl;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-import com.google.inject.Inject;
-import com.google.inject.assistedinject.Assisted;
+import java.util.concurrent.ExecutionException;
+
import org.apache.usergrid.persistence.core.metrics.MetricsFactory;
import org.apache.usergrid.persistence.core.scope.ApplicationScope;
-import org.apache.usergrid.persistence.index.*;
+import org.apache.usergrid.persistence.index.AliasedEntityIndex;
+import org.apache.usergrid.persistence.index.ApplicationEntityIndex;
+import org.apache.usergrid.persistence.index.EntityIndexFactory;
+import org.apache.usergrid.persistence.index.IndexFig;
import org.apache.usergrid.persistence.map.MapManagerFactory;
-import java.util.concurrent.ExecutionException;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.inject.Inject;
/**
* Get index from factory, adds caching
@@ -39,7 +42,6 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
private final IndexFig config;
private final EsProvider provider;
- private final IndexCache indexCache;
private final IndexBufferProducer indexBatchBufferProducer;
private final MetricsFactory metricsFactory;
private final MapManagerFactory mapManagerFactory;
@@ -51,19 +53,18 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
CacheBuilder.newBuilder().maximumSize( 1000 ).build( new CacheLoader<ApplicationScope, ApplicationEntityIndex>() {
public ApplicationEntityIndex load( ApplicationScope scope ) {
return new EsApplicationEntityIndexImpl(
- scope,entityIndex,config, indexBatchBufferProducer, provider,indexCache, metricsFactory, mapManagerFactory, indexFig, indexIdentifier
+ scope,entityIndex,config, indexBatchBufferProducer, provider, metricsFactory, mapManagerFactory, indexFig, indexIdentifier
);
}
} );
@Inject
- public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider, final IndexCache indexCache,
+ public EsEntityIndexFactoryImpl( final IndexFig config, final EsProvider provider,
final IndexBufferProducer indexBatchBufferProducer,
final MetricsFactory metricsFactory, final MapManagerFactory mapManagerFactory,
final IndexFig indexFig, final AliasedEntityIndex entityIndex, final FailureMonitorImpl.IndexIdentifier indexIdentifier ){
this.config = config;
this.provider = provider;
- this.indexCache = indexCache;
this.indexBatchBufferProducer = indexBatchBufferProducer;
this.metricsFactory = metricsFactory;
this.mapManagerFactory = mapManagerFactory;
@@ -75,7 +76,7 @@ public class EsEntityIndexFactoryImpl implements EntityIndexFactory{
@Override
- public ApplicationEntityIndex createApplicationEntityIndex(@Assisted final ApplicationScope appScope) {
+ public ApplicationEntityIndex createApplicationEntityIndex(final ApplicationScope appScope) {
try{
return eiCache.get(appScope);
}catch (ExecutionException ee){
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index 2ea6774..096bd17 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -25,7 +25,10 @@ import java.net.UnknownHostException;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
import org.elasticsearch.common.settings.ImmutableSettings;
+import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
+import org.elasticsearch.node.Node;
+import org.elasticsearch.node.NodeBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +53,7 @@ public class EsProvider {
public static String LOCAL_ES_PORT_PROPNAME = "EMBEDDED_ES_PORT";
+
@Inject
public EsProvider( IndexFig fig ) {
this.indexFig = fig;
@@ -62,37 +66,67 @@ public class EsProvider {
public Client getClient() {
if ( client == null ) {
//synchronize on creating the client so we don't create too many
- createClient( indexFig );
+ createClient();
}
return client;
}
/**
- * Reset the client instnace
+ * Reset the client instance
*/
- public void releaseClient() {
+ public synchronized void releaseClient() {
//reset our static variables
if ( client != null ) {
- client = null;
+ try {
+ client.close();
+ }
+ //if we fail for any reason, null it so the next request creates a new client
+ finally {
+ client = null;
+ }
}
}
- private synchronized void createClient( IndexFig fig ) {
+ /**
+ * Create our client
+ */
+ private synchronized void createClient() {
- if ( client != null) {
+ if ( client != null ) {
return;
}
- final String clusterName = fig.getClusterName();
- final int port = fig.getPort();
- ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder()
- .put( "cluster.name", clusterName )
- .put( "client.transport.sniff", true );
+ final ClientType clientType = ClientType.valueOf( indexFig.getClientType() );
+
+ switch ( clientType ) {
+ case NODE:
+ client = createNodeClient();
+ break;
+
+ case TRANSPORT:
+ client = createTransportClient();
+ break;
+ default:
+ throw new RuntimeException( "Only client types of NODE and TRANSPORT are supported" );
+ }
+ }
+
+
+ /**
+ * Create the transport client
+ * @return
+ */
+ private Client createTransportClient() {
+ final String clusterName = indexFig.getClusterName();
+ final int port = indexFig.getPort();
+
+ ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName )
+ .put( "client.transport.sniff", true );
- String nodeName = fig.getNodeName();
+ String nodeName = indexFig.getNodeName();
if ( "default".equals( nodeName ) ) {
// no nodeName was specified, use hostname
@@ -105,17 +139,76 @@ public class EsProvider {
}
}
- settings.put( "node.name", nodeName);
+ settings.put( "node.name", nodeName );
+
TransportClient transportClient = new TransportClient( settings.build() );
- // we will connect to ES on all configured hosts
- for ( String host : fig.getHosts().split( "," ) ) {
- transportClient.addTransportAddress( new InetSocketTransportAddress(host, port));
- }
- client = transportClient;
+ // we will connect to ES on all configured hosts
+ for ( String host : indexFig.getHosts().split( "," ) ) {
+ transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) );
+ }
+
+ return transportClient;
}
+ /**
+ * Create a node client
+ * @return
+ */
+ public Client createNodeClient() {
+
+ // we will connect to ES on all configured hosts
+
+
+ final String clusterName = indexFig.getClusterName();
+ final String nodeName = indexFig.getNodeName();
+ final int port = indexFig.getPort();
+
+ /**
+ * Create our hosts
+ */
+ final StringBuffer hosts = new StringBuffer();
+
+ for ( String host : indexFig.getHosts().split( "," ) ) {
+ hosts.append( host ).append( ":" ).append( port ).append( "," );
+ }
+
+ //remove the last comma
+ hosts.deleteCharAt( hosts.length() - 1 );
+
+ final String hostString = hosts.toString();
+
+
+ Settings settings = ImmutableSettings.settingsBuilder()
+
+ .put( "cluster.name", clusterName )
+ // this assumes that we're using zen for host discovery. Putting an
+ // explicit set of bootstrap hosts ensures we connect to a valid cluster.
+ .put( "discovery.zen.ping.unicast.hosts", hostString )
+ .put( "discovery.zen.ping.multicast.enabled", "false" ).put( "http.enabled", false )
+
+ .put( "client.transport.ping_timeout", 2000 ) // milliseconds
+ .put( "client.transport.nodes_sampler_interval", 100 ).put( "network.tcp.blocking", true )
+ .put( "node.client", true ).put( "node.name", nodeName )
+
+ .build();
+
+ log.debug( "Creating ElasticSearch client with settings: {}", settings.getAsMap() );
+
+ Node node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).data( false ).node();
+
+ return node.client();
+ }
+
+
+ /**
+ *
+ */
+ public enum ClientType {
+ TRANSPORT,
+ NODE
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
index 0b88dc5..299b95b 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/IndexingUtils.java
@@ -44,7 +44,7 @@ public class IndexingUtils {
public static final String ES_ENTITY_TYPE = "entity";
/**
- * Reserved UG fields in the document
+ * Reserved UG fields in the document
*/
public static final String APPLICATION_ID_FIELDNAME = "applicationId";
@@ -79,6 +79,19 @@ public class IndexingUtils {
public static final String FIELD_LOCATION = "location";
public static final String FIELD_STRING = "string";
+
+ /**
+ * All possible sort values
+ */
+ public static final String SORT_FIELD_BOOLEAN = ENTITY_FIELDS + ".boolean";
+ public static final String SORT_FIELD_INT = ENTITY_FIELDS + ".int";
+ public static final String SORT_FIELD_LONG = ENTITY_FIELDS + ".long";
+ public static final String SORT_FIELD_FLOAT = ENTITY_FIELDS + ".float";
+ public static final String SORT_FIELD_DOUBLE = ENTITY_FIELDS + ".double";
+ public static final String SORT_FIELD_LOCATION = ENTITY_FIELDS + ".location";
+ public static final String SORT_FIELD_STRING = ENTITY_FIELDS + ".string";
+
+
//The value appended to the string field when it's an exact match. Should only be used on search, never index
public static final String FIELD_STRING_EQUALS = FIELD_STRING + ".exact";
@@ -154,8 +167,6 @@ public class IndexingUtils {
/**
* Parse the document id into a candidate result
- * @param documentId
- * @return
*/
public static CandidateResult parseIndexDocId( final String documentId ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 18d4228..1e0ecde 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -23,8 +23,10 @@ package org.apache.usergrid.persistence.index.impl;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.FilterBuilder;
+import org.elasticsearch.index.query.FilterBuilders;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermFilterBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
@@ -44,20 +46,23 @@ import com.google.common.base.Preconditions;
import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+
/**
* The strategy for creating a search request from a parsed query
*/
public class SearchRequestBuilderStrategy {
- private static final Logger logger = LoggerFactory.getLogger(SearchRequestBuilderStrategy.class);
+ private static final Logger logger = LoggerFactory.getLogger( SearchRequestBuilderStrategy.class );
private final EsProvider esProvider;
private final ApplicationScope applicationScope;
private final IndexAlias alias;
private final int cursorTimeout;
- public SearchRequestBuilderStrategy(final EsProvider esProvider, final ApplicationScope applicationScope, final IndexAlias alias, int cursorTimeout){
+
+ public SearchRequestBuilderStrategy( final EsProvider esProvider, final ApplicationScope applicationScope,
+ final IndexAlias alias, int cursorTimeout ) {
this.esProvider = esProvider;
this.applicationScope = applicationScope;
@@ -65,94 +70,75 @@ public class SearchRequestBuilderStrategy {
this.cursorTimeout = cursorTimeout;
}
- public SearchRequestBuilder getBuilder(final SearchEdge searchEdge, final SearchTypes searchTypes, final ParsedQuery query, final int limit) {
- Preconditions.checkArgument(limit <= EntityIndex.MAX_LIMIT, "limit is greater than max "+ EntityIndex.MAX_LIMIT);
+ public SearchRequestBuilder getBuilder( final SearchEdge searchEdge, final SearchTypes searchTypes,
+ final ParsedQuery query, final int limit ) {
- SearchRequestBuilder srb = esProvider.getClient().prepareSearch(alias.getReadAlias())
- .setTypes(searchTypes.getTypeNames(applicationScope))
- .setScroll(cursorTimeout + "m")
- .setQuery(createQueryBuilder( searchEdge,query));
+ Preconditions
+ .checkArgument( limit <= EntityIndex.MAX_LIMIT, "limit is greater than max " + EntityIndex.MAX_LIMIT );
- final FilterBuilder fb = createFilterBuilder(query);
+ SearchRequestBuilder srb = esProvider.getClient().prepareSearch( alias.getReadAlias() )
+ .setTypes( searchTypes.getTypeNames( applicationScope ) )
+ .setScroll( cursorTimeout + "m" );
- //we have post filters, apply them
- if (fb != null) {
- logger.debug(" Filter: {} ", fb.toString());
- srb = srb.setPostFilter(fb);
- }
+ final QueryVisitor visitor = visitParsedQuery(query);
+
+ srb.setQuery( createQueryBuilder( searchEdge, visitor ) );
+
+ final FilterBuilder fb = visitor.getFilterBuilder();
+
+ srb.setPostFilter( fb );
- srb = srb.setFrom(0).setSize(limit);
-
- for (SortPredicate sp : query.getSortPredicates()) {
- throw new RuntimeException( "Fix me" );
-
-// final SortOrder order;
-// if (sp.getDirection().equals( SortPredicate.SortDirection.ASCENDING)) {
-// order = SortOrder.ASC;
-// } else {
-// order = SortOrder.DESC;
-// }
-//
-// // we do not know the type of the "order by" property and so we do not know what
-// // type prefix to use. So, here we add an order by clause for every possible type
-// // that you can order by: string, number and boolean and we ask ElasticSearch
-// // to ignore any fields that are not present.
-//
-// final String stringFieldName = STRING_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder stringSort = SortBuilders.fieldSort(stringFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(stringSort);
-//
-// logger.debug(" Sort: {} order by {}", stringFieldName, order.toString());
-//
-// final String longFieldName = LONG_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder longSort = SortBuilders.fieldSort(longFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(longSort);
-// logger.debug(" Sort: {} order by {}", longFieldName, order.toString());
-//
-//
-// final String doubleFieldName = DOUBLE_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder doubleSort = SortBuilders.fieldSort(doubleFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(doubleSort);
-// logger.debug(" Sort: {} order by {}", doubleFieldName, order.toString());
-//
-//
-// final String booleanFieldName = BOOLEAN_PREFIX + sp.getPropertyName();
-// final FieldSortBuilder booleanSort = SortBuilders.fieldSort(booleanFieldName)
-// .order(order).ignoreUnmapped(true);
-// srb.addSort(booleanSort);
-// logger.debug(" Sort: {} order by {}", booleanFieldName, order.toString());
+
+ srb = srb.setFrom( 0 ).setSize( limit );
+
+ //no sort predicates, sort by edge time descending, entity id second
+ if ( query.getSortPredicates().size() == 0 ) {
+ //sort by the edge timestamp
+ srb.addSort( SortBuilders.fieldSort( IndexingUtils.EDGE_TIMESTAMP_FIELDNAME ).order( SortOrder.DESC ) );
+
+ //sort by the entity id if our times are equal
+ srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) );
}
- return srb;
- }
+ //we have sort predicates, sort them
+ for ( SortPredicate sp : query.getSortPredicates() ) {
- public QueryBuilder createQueryBuilder(final SearchEdge searchEdge, final ParsedQuery query) {
- String context = createContextName(applicationScope, searchEdge );
- QueryBuilder queryBuilder = null;
+ // we do not know the type of the "order by" property and so we do not know what
+ // type prefix to use. So, here we add an order by clause for every possible type
+ // that you can order by: string, number and boolean and we ask ElasticSearch
+ // to ignore any fields that are not present.
- //we have a root operand. Translate our AST into an ES search
- if ( query.getRootOperand() != null ) {
- // In the case of geo only queries, this will return null into the query builder.
- // Once we start using tiles, we won't need this check any longer, since a geo query
- // will return a tile query + post filter
- QueryVisitor v = new EsQueryVistor();
+ final SortOrder order = sp.getDirection().toEsSort();
+ final String propertyName = sp.getPropertyName();
- try {
- query.getRootOperand().visit( v );
- }
- catch ( IndexException ex ) {
- throw new RuntimeException( "Error building ElasticSearch query", ex );
- }
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_STRING, propertyName ) );
+
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_INT, propertyName ) );
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_DOUBLE, propertyName ) );
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_BOOLEAN, propertyName ) );
+
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_LONG, propertyName ) );
+
+ srb.addSort( createSort( order, IndexingUtils.SORT_FIELD_FLOAT, propertyName ) );
- queryBuilder = v.getQueryBuilder();
}
+ return srb;
+ }
+
+
+ public QueryBuilder createQueryBuilder( final SearchEdge searchEdge, final QueryVisitor visitor ) {
+ String context = createContextName( applicationScope, searchEdge );
+
+ QueryBuilder queryBuilder = visitor.getQueryBuilder();
// Add our filter for context to our query for fast execution.
@@ -165,7 +151,7 @@ public class SearchRequestBuilderStrategy {
//make sure we have entity in the context
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
- boolQueryBuilder.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
+ boolQueryBuilder.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
boolQueryBuilder.must( queryBuilder );
@@ -173,21 +159,44 @@ public class SearchRequestBuilderStrategy {
}
- public FilterBuilder createFilterBuilder(ParsedQuery query) {
- FilterBuilder filterBuilder = null;
- if ( query.getRootOperand() != null ) {
- QueryVisitor v = new EsQueryVistor();
- try {
- query.getRootOperand().visit( v );
- } catch ( IndexException ex ) {
+ /**
+ * Perform our visit of the query once for efficiency
+ * @param parsedQuery
+ * @return
+ */
+ private QueryVisitor visitParsedQuery(final ParsedQuery parsedQuery){
+ QueryVisitor v = new EsQueryVistor();
+
+ if ( parsedQuery.getRootOperand() != null ) {
+
+ try {
+ parsedQuery.getRootOperand().visit( v );
+ }
+ catch ( IndexException ex ) {
throw new RuntimeException( "Error building ElasticSearch query", ex );
}
- filterBuilder = v.getFilterBuilder();
+
}
- return filterBuilder;
+ return v;
}
+
+ /**
+ * Create a sort for the property name and field name specified
+ * @param sortOrder The sort order
+ * @param fieldName The name of the field for the type
+ * @param propertyName The property name the user specified for the sort
+ * @return
+ */
+ private FieldSortBuilder createSort( final SortOrder sortOrder, final String fieldName,
+ final String propertyName ) {
+
+ final TermFilterBuilder propertyFilter = FilterBuilders.termFilter( IndexingUtils.FIELD_NAME, propertyName );
+
+
+ return SortBuilders.fieldSort( fieldName ).order( sortOrder ).ignoreUnmapped( true ).setNestedFilter( propertyFilter );
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
index 70d0b30..92636fb 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/SortPredicate.java
@@ -26,6 +26,8 @@ package org.apache.usergrid.persistence.index.query;
import java.io.Serializable;
+import org.elasticsearch.search.sort.SortOrder;
+
/**
* An object that represents a sort predicate
@@ -100,7 +102,23 @@ public final class SortPredicate implements Serializable {
public enum SortDirection {
- ASCENDING, DESCENDING;
+ ASCENDING(SortOrder.ASC), DESCENDING(SortOrder.DESC);
+
+ private final SortOrder esOrder;
+
+
+ SortDirection( final SortOrder esOrder ) {this.esOrder = esOrder;}
+
+
+ /**
+ * Get the ES sort direction
+ * @return
+ */
+ public SortOrder toEsSort(){
+ return esOrder;
+ }
+
+
public static SortDirection find( String s ) {
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
index d295c92..5fc1304 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
@@ -38,65 +38,66 @@ public interface QueryVisitor {
* @param op
* @throws IndexException
*/
- public void visit( AndOperand op ) throws IndexException;
+ void visit( AndOperand op ) throws IndexException;
/**
* @param op
* @throws IndexException
*/
- public void visit( OrOperand op ) throws IndexException;
+ void visit( OrOperand op ) throws IndexException;
/**
* @param op
* @throws IndexException
*/
- public void visit( NotOperand op ) throws IndexException;
+ void visit( NotOperand op ) throws IndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( LessThan op ) throws NoIndexException;
+ void visit( LessThan op ) throws NoIndexException;
/**
* @param op
* @throws NoFullTextIndexException
*/
- public void visit( ContainsOperand op ) throws NoFullTextIndexException;
+ void visit( ContainsOperand op ) throws NoFullTextIndexException;
/**
* @param op
*/
- public void visit( WithinOperand op );
+ void visit( WithinOperand op );
/**
* @param op
* @throws NoIndexException
*/
- public void visit( LessThanEqual op ) throws NoIndexException;
+ void visit( LessThanEqual op ) throws NoIndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( Equal op ) throws NoIndexException;
+ void visit( Equal op ) throws NoIndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( GreaterThan op ) throws NoIndexException;
+ void visit( GreaterThan op ) throws NoIndexException;
/**
* @param op
* @throws NoIndexException
*/
- public void visit( GreaterThanEqual op ) throws NoIndexException;
+ void visit( GreaterThanEqual op ) throws NoIndexException;
/**
* Returns resulting query builder.
*/
- public QueryBuilder getQueryBuilder();
+ QueryBuilder getQueryBuilder();
- public FilterBuilder getFilterBuilder();
+
+ FilterBuilder getFilterBuilder();
}
http://git-wip-us.apache.org/repos/asf/incubator-usergrid/blob/cb9b06a3/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
index ada7420..0ac0a59 100644
--- a/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
+++ b/stack/corepersistence/queryindex/src/test/java/org/apache/usergrid/persistence/query/tree/GrammarTreeTest.java
@@ -436,11 +436,7 @@ public class GrammarTreeTest {
assertEquals( 37f, withinOperand.getLatitude().getFloatValue(), 0 );
assertEquals( -75f, withinOperand.getLongitude().getFloatValue(), 0 );
- SearchRequestBuilderStrategy builderStrategy =
- new SearchRequestBuilderStrategy( null, new ApplicationScopeImpl( new SimpleId( "test" ) ), null, 100 );
- QueryBuilder qb =
- builderStrategy.createQueryBuilder( new SearchEdgeImpl( new SimpleId( "owner" ), "app",
- SearchEdge.NodeType.SOURCE ), query );
+
}