You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@usergrid.apache.org by md...@apache.org on 2017/11/20 17:55:58 UTC
[3/3] usergrid git commit: Upgrade Elastic search t0 5.6.3
Upgrade Elastic search t0 5.6.3
Project: http://git-wip-us.apache.org/repos/asf/usergrid/repo
Commit: http://git-wip-us.apache.org/repos/asf/usergrid/commit/5465d807
Tree: http://git-wip-us.apache.org/repos/asf/usergrid/tree/5465d807
Diff: http://git-wip-us.apache.org/repos/asf/usergrid/diff/5465d807
Branch: refs/heads/3.0.0_experimental
Commit: 5465d807ef3cbe8521b623b9d520be808745daa3
Parents: c172133
Author: Peter Johnson <pj...@apigee.com>
Authored: Tue Oct 24 16:07:37 2017 -0700
Committer: Mike Dunker <md...@google.com>
Committed: Mon Nov 20 09:51:13 2017 -0800
----------------------------------------------------------------------
stack/corepersistence/pom.xml | 2 +-
stack/corepersistence/queryindex/pom.xml | 6 ++
.../index/impl/EsEntityIndexImpl.java | 66 ++++++------
.../index/impl/EsIndexProducerImpl.java | 23 ++++-
.../persistence/index/impl/EsProvider.java | 18 ++--
.../persistence/index/impl/EsQueryVistor.java | 100 ++++++++-----------
.../index/impl/NoOpFilterBuilder.java | 13 +--
.../index/impl/NoOpQueryBuilder.java | 33 +++++-
.../impl/SearchRequestBuilderStrategy.java | 36 ++++---
.../persistence/index/impl/SortBuilder.java | 8 +-
.../index/query/tree/QueryVisitor.java | 10 +-
.../persistence/index/usergrid-mappings.json | 2 -
12 files changed, 174 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/pom.xml b/stack/corepersistence/pom.xml
index 45aad4b..ff09cf2 100644
--- a/stack/corepersistence/pom.xml
+++ b/stack/corepersistence/pom.xml
@@ -82,7 +82,7 @@ limitations under the License.
<commons.io.version>2.4</commons.io.version>
<commons.lang.version>3.1</commons.lang.version>
<datastax.version>2.1.10.3</datastax.version>
- <elasticsearch.version>1.7.5</elasticsearch.version>
+ <elasticsearch.version>5.6.3</elasticsearch.version>
<fasterxml-uuid.version>3.1.3</fasterxml-uuid.version>
<guava.version>18.0</guava.version>
<guice.version>4.0-beta5</guice.version>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/pom.xml
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/pom.xml b/stack/corepersistence/queryindex/pom.xml
index 11856cb..c5401e4 100644
--- a/stack/corepersistence/queryindex/pom.xml
+++ b/stack/corepersistence/queryindex/pom.xml
@@ -101,6 +101,12 @@
<type>jar</type>
</dependency>
+ <dependency>
+ <groupId>org.elasticsearch.client</groupId>
+ <artifactId>transport</artifactId>
+ <version>${elasticsearch.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.elasticsearch</groupId>
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
index 211cf70..9a0af48 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsEntityIndexImpl.java
@@ -47,10 +47,8 @@ import org.apache.usergrid.persistence.index.query.tree.QueryVisitor;
import org.apache.usergrid.persistence.index.utils.IndexValidationUtils;
import org.apache.usergrid.persistence.model.entity.Id;
import org.apache.usergrid.persistence.model.util.UUIDGenerator;
-import org.elasticsearch.action.ActionFuture;
-import org.elasticsearch.action.ActionListener;
-import org.elasticsearch.action.ListenableActionFuture;
-import org.elasticsearch.action.ShardOperationFailedException;
+import org.elasticsearch.ResourceAlreadyExistsException;
+import org.elasticsearch.action.*;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequestBuilder;
@@ -59,22 +57,21 @@ import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse;
import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
import org.elasticsearch.action.admin.indices.stats.CommonStats;
import org.elasticsearch.action.admin.indices.stats.IndicesStatsResponse;
-import org.elasticsearch.action.deletebyquery.DeleteByQueryResponse;
-import org.elasticsearch.action.deletebyquery.IndexDeleteByQueryResponse;
+import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.AdminClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.index.IndexNotFoundException;
import org.elasticsearch.index.query.*;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.indices.IndexAlreadyExistsException;
-import org.elasticsearch.indices.IndexMissingException;
+import org.elasticsearch.index.reindex.BulkByScrollResponse;
+import org.elasticsearch.index.reindex.DeleteByQueryAction;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.metrics.sum.Sum;
-import org.elasticsearch.search.aggregations.metrics.sum.SumBuilder;
+import org.elasticsearch.search.aggregations.metrics.sum.SumAggregationBuilder;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -206,13 +203,13 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
//Create index
try {
final AdminClient admin = esProvider.getClient().admin();
- Settings settings = ImmutableSettings.settingsBuilder()
+ Settings settings = Settings.builder()
.put("index.number_of_shards", numberOfShards)
.put("index.number_of_replicas", numberOfReplicas)
//dont' allow unmapped queries, and don't allow dynamic mapping
.put("index.query.parse.allow_unmapped_fields", false)
.put("index.mapper.dynamic", false)
- .put("action.write_consistency", writeConsistency)
+ // .put("action.write_consistency", writeConsistency)
.build();
//Added For Graphite Metrics
@@ -230,7 +227,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
logger.info("Created new Index Name [{}] ACK=[{}]", indexName, cir.isAcknowledged());
- } catch (IndexAlreadyExistsException e) {
+ } catch (ResourceAlreadyExistsException e) {
logger.info("Index Name [{}] already exists", indexName);
}
/**
@@ -244,7 +241,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
testNewIndex();
- } catch (IndexAlreadyExistsException expected) {
+ } catch (ResourceAlreadyExistsException expected) {
// this is expected to happen if index already exists, it's a no-op and swallow
} catch (IOException e) {
throw new RuntimeException("Unable to initialize index", e);
@@ -452,8 +449,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
for (SortPredicate sortPredicate : parsedQuery.getSortPredicates() ){
hasGeoSortPredicates = visitor.getGeoSorts().contains(sortPredicate.getPropertyName());
}
-
-
+
final String cacheKey = applicationScope.getApplication().getUuid().toString()+"_"+searchEdge.getEdgeName();
final Object totalEdgeSizeFromCache = sizeCache.getIfPresent(cacheKey);
long totalEdgeSizeInBytes;
@@ -498,6 +494,9 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
final Timer.Context timerContext = searchTimer.time();
try {
+ if (logger.isDebugEnabled()) {
+ logger.debug("Query to execute = {}", srb.toString());
+ }
searchResponse = srb.execute().actionGet();
}
@@ -594,21 +593,25 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
//Added For Graphite Metrics
return Observable.from( indexes ).flatMap( index -> {
- final ListenableActionFuture<DeleteByQueryResponse> response =
- esProvider.getClient().prepareDeleteByQuery( alias.getWriteAlias() ).setQuery( tqb ).execute();
+ ListenableActionFuture<BulkByScrollResponse> response =
+ DeleteByQueryAction.INSTANCE.newRequestBuilder( esProvider.getClient())
+ .filter(tqb)
+ .source(indexes)
+ .execute();
+
- response.addListener( new ActionListener<DeleteByQueryResponse>() {
+ response.addListener( new ActionListener<BulkByScrollResponse>() {
@Override
- public void onResponse( DeleteByQueryResponse response ) {
+ public void onResponse( BulkByScrollResponse response ) {
checkDeleteByQueryResponse( tqb, response );
}
-
@Override
- public void onFailure( Throwable e ) {
+ public void onFailure(Exception e) {
logger.error( "Failed on delete index", e.getMessage() );
}
+
} );
return Observable.from( response );
} ).doOnError( t -> logger.error( "Failed on delete application", t.getMessage() ) );
@@ -618,17 +621,14 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
/**
* Validate the response doesn't contain errors, if it does, fail fast at the first error we encounter
*/
- private void checkDeleteByQueryResponse( final QueryBuilder query, final DeleteByQueryResponse response ) {
+ private void checkDeleteByQueryResponse( final QueryBuilder query, final BulkByScrollResponse response ) {
- for ( IndexDeleteByQueryResponse indexDeleteByQueryResponse : response ) {
- final ShardOperationFailedException[] failures = indexDeleteByQueryResponse.getFailures();
+ List<BulkItemResponse.Failure> failures = response.getBulkFailures();
- for ( ShardOperationFailedException failedException : failures ) {
- logger.error("Unable to delete by query {}. Failed with code {} and reason {} on shard {} in index {}",
- query.toString(),
- failedException.status().getStatus(), failedException.reason(),
- failedException.shardId(), failedException.index() );
- }
+
+ for ( BulkItemResponse.Failure failure : failures ) {
+ logger.error("Unable to delete by query {}. Failed with code {} and reason {} in index {}",
+ query.toString(), failure.getStatus() , failure.getMessage(), failure.getIndex());
}
}
@@ -818,7 +818,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
.actionGet();
final CommonStats indexStats = statsResponse.getIndex(indexName).getTotal();
indexSize = indexStats.getStore().getSizeInBytes();
- } catch (IndexMissingException e) {
+ } catch (IndexNotFoundException e) {
// if for some reason the index size does not exist,
// log an error and we can assume size is 0 as it doesn't exist
logger.error("Unable to get size for index {} due to IndexMissingException for app {}",
@@ -837,7 +837,7 @@ public class EsEntityIndexImpl implements EntityIndex,VersionedData {
private long getEntitySizeAggregation( final SearchRequestBuilder builder ) {
final String key = "entitySize";
- SumBuilder sumBuilder = new SumBuilder(key);
+ SumAggregationBuilder sumBuilder = new SumAggregationBuilder(key);
sumBuilder.field("entitySize");
builder.addAggregation(sumBuilder);
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
index 8f58ef7..468a99e 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsIndexProducerImpl.java
@@ -26,10 +26,10 @@ import com.codahale.metrics.Histogram;
import org.apache.usergrid.persistence.index.EntityIndexBatch;
-import org.elasticsearch.action.WriteConsistencyLevel;
import org.elasticsearch.action.bulk.BulkItemResponse;
import org.elasticsearch.action.bulk.BulkRequestBuilder;
import org.elasticsearch.action.bulk.BulkResponse;
+import org.elasticsearch.action.support.ActiveShardCount;
import org.elasticsearch.client.Client;
import org.elasticsearch.rest.RestStatus;
import org.slf4j.Logger;
@@ -169,9 +169,26 @@ public class EsIndexProducerImpl implements IndexProducer {
* initialize request
*/
private BulkRequestBuilder initRequest() {
+
BulkRequestBuilder bulkRequest = client.prepareBulk();
- bulkRequest.setConsistencyLevel( WriteConsistencyLevel.fromString( config.getWriteConsistencyLevel() ) );
- bulkRequest.setRefresh( config.isForcedRefresh() );
+
+ String refreshPolicyConfig = String.valueOf(config.isForcedRefresh());
+ bulkRequest.setRefreshPolicy(refreshPolicyConfig);
+
+ String consistencyLevel = config.getWriteConsistencyLevel();
+
+ if ("one".equals(consistencyLevel)) {
+ bulkRequest.setWaitForActiveShards(1);
+ }
+
+ if ("all".equals(consistencyLevel)) {
+ bulkRequest.setWaitForActiveShards(ActiveShardCount.ALL);
+ }
+
+ if ("none".equals(consistencyLevel)) {
+ bulkRequest.setWaitForActiveShards(ActiveShardCount.NONE);
+ }
+
return bulkRequest;
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
index e81aa0f..fe35a02 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsProvider.java
@@ -24,11 +24,10 @@ import java.net.UnknownHostException;
import org.elasticsearch.client.Client;
import org.elasticsearch.client.transport.TransportClient;
-import org.elasticsearch.common.settings.ImmutableSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.transport.InetSocketTransportAddress;
import org.elasticsearch.node.Node;
-import org.elasticsearch.node.NodeBuilder;
+import org.elasticsearch.transport.client.PreBuiltTransportClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -123,7 +122,8 @@ public class EsProvider {
final String clusterName = indexFig.getClusterName();
final int port = indexFig.getPort();
- ImmutableSettings.Builder settings = ImmutableSettings.settingsBuilder().put( "cluster.name", clusterName )
+ Settings.Builder settings = Settings.builder()
+ .put( "cluster.name", clusterName )
.put( "client.transport.sniff", true );
String nodeName = indexFig.getNodeName();
@@ -142,11 +142,15 @@ public class EsProvider {
settings.put( "node.name", nodeName );
- TransportClient transportClient = new TransportClient( settings.build() );
+ TransportClient transportClient = new PreBuiltTransportClient(settings.build());
// we will connect to ES on all configured hosts
for ( String host : indexFig.getHosts().split( "," ) ) {
- transportClient.addTransportAddress( new InetSocketTransportAddress( host, port ) );
+ try {
+ transportClient.addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName(host), port));
+ } catch (UnknownHostException uhe) {
+ logger.error( "Couldn't resolve hostname {} to use as ES node ", host );
+ }
}
return transportClient;
@@ -181,7 +185,7 @@ public class EsProvider {
final String hostString = hosts.toString();
- Settings settings = ImmutableSettings.settingsBuilder()
+ Settings settings = Settings.builder()
.put( "cluster.name", clusterName )
@@ -200,7 +204,7 @@ public class EsProvider {
logger.trace("Creating ElasticSearch client with settings: {}", settings.getAsMap());
}
- Node node = NodeBuilder.nodeBuilder().settings( settings ).client( true ).data( false ).node();
+ Node node = new Node( settings );
return node.client();
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
index 4dd0d24..7adb332 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/EsQueryVistor.java
@@ -22,20 +22,11 @@ package org.apache.usergrid.persistence.index.impl;
import java.util.Stack;
import java.util.UUID;
+import org.apache.lucene.search.join.ScoreMode;
import org.elasticsearch.common.geo.GeoDistance;
import org.elasticsearch.common.unit.DistanceUnit;
-import org.elasticsearch.index.query.BoolFilterBuilder;
-import org.elasticsearch.index.query.BoolQueryBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.MatchQueryBuilder;
-import org.elasticsearch.index.query.NestedFilterBuilder;
-import org.elasticsearch.index.query.NestedQueryBuilder;
-import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.QueryBuilders;
-import org.elasticsearch.index.query.RangeFilterBuilder;
-import org.elasticsearch.index.query.TermFilterBuilder;
-import org.elasticsearch.index.query.WildcardQueryBuilder;
+import org.elasticsearch.index.query.*;
+
import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.slf4j.Logger;
@@ -75,7 +66,7 @@ public class EsQueryVistor implements QueryVisitor {
/**
* Our queryBuilders for filter operations
*/
- private final Stack<FilterBuilder> filterBuilders = new Stack<>();
+ private final Stack<QueryBuilder> filterBuilders = new Stack<>();
private final GeoSortFields geoSortFields = new GeoSortFields();
@@ -89,12 +80,12 @@ public class EsQueryVistor implements QueryVisitor {
//get all the right
final QueryBuilder rightQuery = queryBuilders.pop();
- final FilterBuilder rightFilter = filterBuilders.pop();
+ final QueryBuilder rightFilter = filterBuilders.pop();
//get all the left
final QueryBuilder leftQuery = queryBuilders.pop();
- final FilterBuilder leftFilter = filterBuilders.pop();
+ final QueryBuilder leftFilter = filterBuilders.pop();
//push our boolean filters
@@ -131,7 +122,7 @@ public class EsQueryVistor implements QueryVisitor {
//use left and right
if ( useLeftFilter && useRightFilter ) {
- final BoolFilterBuilder fb = FilterBuilders.boolFilter().must(leftFilter).must(rightFilter);
+ final BoolQueryBuilder fb = QueryBuilders.boolQuery().must(leftFilter).must(rightFilter);
filterBuilders.push( fb );
}
@@ -157,12 +148,12 @@ public class EsQueryVistor implements QueryVisitor {
op.getRight().visit( this );
final QueryBuilder rightQuery = queryBuilders.pop();
- final FilterBuilder rightFilter = filterBuilders.pop();
+ final QueryBuilder rightFilter = filterBuilders.pop();
//get all the left
final QueryBuilder leftQuery = queryBuilders.pop();
- final FilterBuilder leftFilter = filterBuilders.pop();
+ final QueryBuilder leftFilter = filterBuilders.pop();
final boolean useLeftQuery = use( leftQuery );
@@ -193,7 +184,7 @@ public class EsQueryVistor implements QueryVisitor {
//use left and right
if ( useLeftFilter && useRightFilter ) {
- final BoolFilterBuilder fb = FilterBuilders.boolFilter().should( leftFilter ).should( rightFilter );
+ final BoolQueryBuilder fb = QueryBuilders.boolQuery().should( leftFilter ).should( rightFilter );
filterBuilders.push( fb );
}
@@ -232,12 +223,12 @@ public class EsQueryVistor implements QueryVisitor {
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
}
- final FilterBuilder notFilterBuilder = filterBuilders.pop();
+ final QueryBuilder notFilterBuilder = filterBuilders.pop();
//push the filter in
if ( use( notFilterBuilder ) ) {
- final FilterBuilder notFilter = FilterBuilders.boolFilter().mustNot( notFilterBuilder ) ;
+ final QueryBuilder notFilter = QueryBuilders.boolQuery().mustNot( notFilterBuilder ) ;
//just the root node
if(!rootNode) {
@@ -245,7 +236,7 @@ public class EsQueryVistor implements QueryVisitor {
}
//not the root node, we have to select all to subtract from with the NOT statement
else{
- final FilterBuilder selectAllFilter = FilterBuilders.boolFilter().must( FilterBuilders.matchAllFilter()) .must( notFilter );
+ final QueryBuilder selectAllFilter = QueryBuilders.boolQuery().must( QueryBuilders.matchAllQuery()) .must( notFilter );
filterBuilders.push( selectAllFilter );
}
@@ -293,9 +284,10 @@ public class EsQueryVistor implements QueryVisitor {
float distance = op.getDistance().getFloatValue();
- final FilterBuilder fb =
- FilterBuilders.geoDistanceFilter( IndexingUtils.FIELD_LOCATION_NESTED ).lat( lat ).lon( lon )
- .distance( distance, DistanceUnit.METERS );
+ final QueryBuilder fb =
+ QueryBuilders.geoDistanceQuery( IndexingUtils.FIELD_LOCATION_NESTED )
+ .point(lat, lon)
+ .distance( distance, DistanceUnit.METERS );
filterBuilders.push( fieldNameTerm( name, fb ) );
@@ -306,10 +298,11 @@ public class EsQueryVistor implements QueryVisitor {
//this geoSort won't has a sort on it
final GeoDistanceSortBuilder geoSort =
- SortBuilders.geoDistanceSort( IndexingUtils.FIELD_LOCATION_NESTED ).unit( DistanceUnit.METERS )
- .geoDistance(GeoDistance.SLOPPY_ARC).point(lat, lon);
+ SortBuilders.geoDistanceSort( IndexingUtils.FIELD_LOCATION_NESTED, lat, lon )
+ .unit( DistanceUnit.METERS )
+ .geoDistance(GeoDistance.ARC).point(lat, lon);
- final TermFilterBuilder sortPropertyName = sortPropertyTermFilter(name);
+ final TermQueryBuilder sortPropertyName = sortPropertyTermFilter(name);
geoSort.setNestedFilter( sortPropertyName );
@@ -328,8 +321,8 @@ public class EsQueryVistor implements QueryVisitor {
final Object value = op.getLiteral().getValue();
- final RangeFilterBuilder termQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).lt(sanitize(value));
+ final RangeQueryBuilder termQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).lt(sanitize(value));
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
@@ -346,8 +339,8 @@ public class EsQueryVistor implements QueryVisitor {
final Object value = op.getLiteral().getValue();
- final RangeFilterBuilder termQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).lte(sanitize(value));
+ final RangeQueryBuilder termQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).lte(sanitize(value));
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
@@ -383,8 +376,8 @@ public class EsQueryVistor implements QueryVisitor {
}
//it's an exact match, use a filter
- final TermFilterBuilder termFilter =
- FilterBuilders.termFilter( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue );
+ final TermQueryBuilder termFilter =
+ QueryBuilders.termQuery( IndexingUtils.FIELD_STRING_NESTED_UNANALYZED, stringValue );
queryBuilders.push( NoOpQueryBuilder.INSTANCE );
filterBuilders.push( fieldNameTerm( name, termFilter ) );
@@ -394,8 +387,8 @@ public class EsQueryVistor implements QueryVisitor {
// assume all other types need prefix
- final TermFilterBuilder termQuery =
- FilterBuilders.termFilter(getFieldNameForType(value), sanitize(value));
+ final TermQueryBuilder termQuery =
+ QueryBuilders.termQuery(getFieldNameForType(value), sanitize(value));
filterBuilders.push( fieldNameTerm( name, termQuery ) );
@@ -409,8 +402,8 @@ public class EsQueryVistor implements QueryVisitor {
final Object value = op.getLiteral().getValue();
- final RangeFilterBuilder rangeQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).gt(sanitize(value));
+ final RangeQueryBuilder rangeQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).gt(sanitize(value));
filterBuilders.push( fieldNameTerm( name, rangeQuery ) );
@@ -424,8 +417,8 @@ public class EsQueryVistor implements QueryVisitor {
Object value = op.getLiteral().getValue();
- final RangeFilterBuilder rangeQuery =
- FilterBuilders.rangeFilter( getFieldNameForType( value ) ).gte(sanitize(value));
+ final RangeQueryBuilder rangeQuery =
+ QueryBuilders.rangeQuery( getFieldNameForType( value ) ).gte(sanitize(value));
filterBuilders.push(fieldNameTerm(name, rangeQuery));
@@ -434,12 +427,12 @@ public class EsQueryVistor implements QueryVisitor {
@Override
- public Optional<FilterBuilder> getFilterBuilder() {
+ public Optional<QueryBuilder> getFilterBuilder() {
if ( filterBuilders.empty() ) {
return Optional.absent();
}
- final FilterBuilder builder = filterBuilders.peek();
+ final QueryBuilder builder = filterBuilders.peek();
if ( !use( builder ) ) {
return Optional.absent();
@@ -484,26 +477,26 @@ public class EsQueryVistor implements QueryVisitor {
booleanQuery.must( fieldValueQuery );
- return QueryBuilders.nestedQuery(IndexingUtils.ENTITY_FIELDS, booleanQuery);
+ return QueryBuilders.nestedQuery(IndexingUtils.ENTITY_FIELDS, booleanQuery, ScoreMode.None);
}
/**
* Generate the field name term for the field name for filters
- */
- private NestedFilterBuilder fieldNameTerm( final String fieldName, final FilterBuilder fieldValueBuilder ) {
+ *
+ private NestedQueryBuilder fieldNameTerm( final String fieldName, final QueryBuilder fieldValueBuilder ) {
- final BoolFilterBuilder booleanQuery = FilterBuilders.boolFilter();
+ final BoolQueryBuilder booleanQuery = QueryBuilders.boolQuery();
- booleanQuery.must( FilterBuilders.termFilter( IndexingUtils.FIELD_NAME_NESTED, fieldName ) );
+ booleanQuery.must( QueryBuilders.termQuery( IndexingUtils.FIELD_NAME_NESTED, fieldName ) );
booleanQuery.must( fieldValueBuilder );
- return FilterBuilders.nestedFilter( IndexingUtils.ENTITY_FIELDS, booleanQuery );
+ return QueryBuilders.nestedQuery( IndexingUtils.ENTITY_FIELDS, booleanQuery );
}
-
+*/
/**
* Get the field name for the primitive type
*/
@@ -578,14 +571,7 @@ public class EsQueryVistor implements QueryVisitor {
* Return false if our element is a no-op, true otherwise
*/
private boolean use( final QueryBuilder queryBuilder ) {
- return queryBuilder != NoOpQueryBuilder.INSTANCE;
+ return queryBuilder != NoOpQueryBuilder.INSTANCE && queryBuilder != NoOpFilterBuilder.INSTANCE;
}
-
- /**
- * Return false if our element is a no-op, true otherwise
- */
- private boolean use( final FilterBuilder filterBuilder ) {
- return filterBuilder != NoOpFilterBuilder.INSTANCE;
- }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
index 9eca433..780ecf3 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpFilterBuilder.java
@@ -20,22 +20,11 @@
package org.apache.usergrid.persistence.index.impl;
-import java.io.IOException;
-
-import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.BaseFilterBuilder;
-
-
/**
* Marker interface that does nothing
*/
-public class NoOpFilterBuilder extends BaseFilterBuilder {
+public class NoOpFilterBuilder extends NoOpQueryBuilder {
public static final NoOpFilterBuilder INSTANCE = new NoOpFilterBuilder();
-
- @Override
- protected void doXContent( final XContentBuilder builder, final Params params ) throws IOException {
- //no op
- }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
index 2498812..69b39ee 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/NoOpQueryBuilder.java
@@ -22,20 +22,49 @@ package org.apache.usergrid.persistence.index.impl;
import java.io.IOException;
+import org.apache.lucene.search.Query;
+import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
-import org.elasticsearch.index.query.BaseQueryBuilder;
+import org.elasticsearch.index.query.AbstractQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
+import org.elasticsearch.index.query.QueryShardContext;
/**
* Marker interface that does nothing
*/
-public class NoOpQueryBuilder extends BaseQueryBuilder implements QueryBuilder {
+public class NoOpQueryBuilder extends AbstractQueryBuilder implements QueryBuilder {
public static final NoOpQueryBuilder INSTANCE = new NoOpQueryBuilder();
@Override
+ protected void doWriteTo(StreamOutput out) throws IOException {
+
+ }
+
+ @Override
protected void doXContent( final XContentBuilder builder, final Params params ) throws IOException {
//no op
}
+
+ @Override
+ protected Query doToQuery(QueryShardContext context) throws IOException {
+ return null;
+ }
+
+ @Override
+ protected boolean doEquals(AbstractQueryBuilder other) {
+ return false;
+ }
+
+ @Override
+ protected int doHashCode() {
+ return 0;
+ }
+
+
+ @Override
+ public String getWriteableName() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
index 5812c6f..3119b4a 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SearchRequestBuilderStrategy.java
@@ -23,11 +23,10 @@ package org.apache.usergrid.persistence.index.impl;
import org.apache.usergrid.persistence.index.IndexAlias;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchType;
-import org.elasticsearch.index.query.BoolFilterBuilder;
-import org.elasticsearch.index.query.FilterBuilder;
-import org.elasticsearch.index.query.FilterBuilders;
+import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.index.query.TermFilterBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.sort.FieldSortBuilder;
import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
@@ -50,7 +49,7 @@ import com.google.common.base.Preconditions;
import java.util.List;
import java.util.Map;
-import static org.apache.usergrid.persistence.index.impl.IndexingUtils.createContextName;
+import static org.apache.usergrid.persistence.index.impl.IndexingUtils.*;
import static org.apache.usergrid.persistence.index.impl.SortBuilder.sortPropertyTermFilter;
@@ -145,6 +144,7 @@ public class SearchRequestBuilderStrategy {
//sort by the entity id if our times are equal
//srb.addSort( SortBuilders.fieldSort( IndexingUtils.ENTITY_ID_FIELDNAME ).order( SortOrder.ASC ) );
+
return;
}
@@ -205,7 +205,7 @@ public class SearchRequestBuilderStrategy {
* Create our filter builder. We need to restrict our results on edge search, as well as on types, and any filters
* that came from the grammar.
*/
- private FilterBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor,
+ private QueryBuilder createFilterBuilder( final SearchEdge searchEdge, final QueryVisitor visitor,
final SearchTypes searchTypes ) {
String context = createContextName( applicationScope, searchEdge );
@@ -218,10 +218,10 @@ public class SearchRequestBuilderStrategy {
// Do we need to put the context term first for performance?
//make sure we have entity in the context
- BoolFilterBuilder boolQueryFilter = FilterBuilders.boolFilter();
+ BoolQueryBuilder boolQueryFilter = QueryBuilders.boolQuery();
//add our edge search
- boolQueryFilter.must( FilterBuilders.termFilter( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
+ boolQueryFilter.must( QueryBuilders.termQuery( IndexingUtils.EDGE_SEARCH_FIELDNAME, context ) );
/**
@@ -231,19 +231,23 @@ public class SearchRequestBuilderStrategy {
if ( sourceTypes.length > 0 ) {
- final FilterBuilder[] typeTerms = new FilterBuilder[sourceTypes.length];
+ final QueryBuilder[] typeTerms = new QueryBuilder[sourceTypes.length];
+
+ BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
for ( int i = 0; i < sourceTypes.length; i++ ) {
- typeTerms[i] = FilterBuilders.termFilter( IndexingUtils.ENTITY_TYPE_FIELDNAME, sourceTypes[i] );
+ typeTerms[i] = QueryBuilders.termQuery( IndexingUtils.ENTITY_TYPE_FIELDNAME, sourceTypes[i] );
+ boolQuery.filter(typeTerms[i]);
}
//add all our types, 1 type must match per query
- boolQueryFilter.must( FilterBuilders.orFilter( typeTerms ) );
+ //boolQueryFilter.must( QueryBuilders.orFilter( typeTerms ) );
+ boolQueryFilter.must( boolQuery );
}
//if we have a filter from our visitor, add it
- Optional<FilterBuilder> queryBuilder = visitor.getFilterBuilder();
+ Optional<QueryBuilder> queryBuilder = visitor.getFilterBuilder();
if ( queryBuilder.isPresent() ) {
boolQueryFilter.must( queryBuilder.get() );
@@ -263,9 +267,13 @@ public class SearchRequestBuilderStrategy {
private FieldSortBuilder createSort( final SortOrder sortOrder, final String fieldName,
final String propertyName ) {
- final TermFilterBuilder propertyFilter = sortPropertyTermFilter( propertyName );
+ final TermQueryBuilder propertyFilter = QueryBuilders.termQuery( ENTITY_FIELDS + "." + IndexingUtils.FIELD_NAME, propertyName );
+ FieldSortBuilder fieldSortBuilder = SortBuilders.fieldSort( fieldName )
+ .order( sortOrder )
+ .setNestedFilter( propertyFilter )
+ .setNestedPath(ENTITY_FIELDS );
- return SortBuilders.fieldSort( fieldName ).order( sortOrder ).setNestedFilter( propertyFilter );
+ return fieldSortBuilder;
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
index ad78be7..f487747 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/impl/SortBuilder.java
@@ -24,8 +24,8 @@
package org.apache.usergrid.persistence.index.impl;
-import org.elasticsearch.index.query.FilterBuilders;
-import org.elasticsearch.index.query.TermFilterBuilder;
+import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.TermQueryBuilder;
/**
@@ -37,8 +37,8 @@ public class SortBuilder {
/**
* Create a term filter for our sorts
*/
- public static TermFilterBuilder sortPropertyTermFilter( final String propertyName ) {
- return FilterBuilders.termFilter( IndexingUtils.FIELD_NAME, propertyName );
+ public static TermQueryBuilder sortPropertyTermFilter( final String propertyName ) {
+ return QueryBuilders.termQuery( IndexingUtils.FIELD_NAME, propertyName );
}
}
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
index 273f23f..5e2d50f 100644
--- a/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
+++ b/stack/corepersistence/queryindex/src/main/java/org/apache/usergrid/persistence/index/query/tree/QueryVisitor.java
@@ -19,19 +19,13 @@
package org.apache.usergrid.persistence.index.query.tree;
-import java.util.Collection;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.usergrid.persistence.index.exceptions.NoFullTextIndexException;
import org.apache.usergrid.persistence.index.exceptions.NoIndexException;
import org.apache.usergrid.persistence.index.exceptions.IndexException;
import org.apache.usergrid.persistence.index.impl.GeoSortFields;
-import org.apache.usergrid.persistence.index.query.SortPredicate;
-import org.elasticsearch.index.query.FilterBuilder;
+
import org.elasticsearch.index.query.QueryBuilder;
-import org.elasticsearch.search.sort.GeoDistanceSortBuilder;
import com.google.common.base.Optional;
@@ -108,7 +102,7 @@ public interface QueryVisitor {
* Return any filters created during parsing
* @return
*/
- Optional<FilterBuilder> getFilterBuilder();
+ Optional<QueryBuilder> getFilterBuilder();
http://git-wip-us.apache.org/repos/asf/usergrid/blob/5465d807/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
----------------------------------------------------------------------
diff --git a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
index 3d2517c..5142d76 100644
--- a/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
+++ b/stack/corepersistence/queryindex/src/main/resources/org/apache/usergrid/persistence/index/usergrid-mappings.json
@@ -73,8 +73,6 @@
},
"location": {
"type": "geo_point",
- "lat_lon": true,
- "geohash": true,
"doc_values": true
},
"string": {