You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by dg...@apache.org on 2018/06/26 14:36:23 UTC
[2/2] incubator-unomi git commit: UNOMI-185 create an additional
endpoint to execute aggregate query,
the new endpoint return a bit less information but is much faster
UNOMI-185 create an additional endpoint to execute aggregate query, the new endpoint return a bit less information but is much faster
Project: http://git-wip-us.apache.org/repos/asf/incubator-unomi/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-unomi/commit/d7e5c2d2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-unomi/tree/d7e5c2d2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-unomi/diff/d7e5c2d2
Branch: refs/heads/feature-optimized-aggregate-query
Commit: d7e5c2d2ff18cc7259eba6c4fef7b4351e790a34
Parents: e467a9e
Author: dgaillard <dg...@jahia.com>
Authored: Tue Jun 26 16:36:09 2018 +0200
Committer: dgaillard <dg...@jahia.com>
Committed: Tue Jun 26 16:36:09 2018 +0200
----------------------------------------------------------------------
.../apache/unomi/api/services/QueryService.java | 14 +++++
.../ElasticSearchPersistenceServiceImpl.java | 64 ++++++++++++++------
.../persistence/spi/PersistenceService.java | 13 ++++
.../BooleanConditionESQueryBuilder.java | 12 +++-
.../apache/unomi/rest/QueryServiceEndPoint.java | 19 ++++++
.../services/services/QueryServiceImpl.java | 61 ++++++++++++-------
6 files changed, 140 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7e5c2d2/api/src/main/java/org/apache/unomi/api/services/QueryService.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/unomi/api/services/QueryService.java b/api/src/main/java/org/apache/unomi/api/services/QueryService.java
index 2465e65..3f3cb7b 100644
--- a/api/src/main/java/org/apache/unomi/api/services/QueryService.java
+++ b/api/src/main/java/org/apache/unomi/api/services/QueryService.java
@@ -44,6 +44,7 @@ public interface QueryService {
*
* Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and aggregated by possible values of the specified
* property or, if the specified query is not {@code null}, perform that aggregate query.
+ * Also return the global count of document matching the {@code ITEM_TYPE}
*
* @param itemType the String representation of the item type we want to retrieve the count of, as defined by its class' {@code ITEM_TYPE} field
* @param property the property we're aggregating on, i.e. for each possible value of this property, we are counting how many items of the specified type have that value
@@ -54,6 +55,19 @@ public interface QueryService {
Map<String, Long> getAggregate(String itemType, String property, AggregateQuery query);
/**
+ * Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and aggregated by possible values of the specified
+ * property or, if the specified query is not {@code null}, perform that aggregate query.
+ * This aggregate won't return the global count and should therefore be much faster than {@link #getAggregate(String, String, AggregateQuery)}
+ *
+ * @param itemType the String representation of the item type we want to retrieve the count of, as defined by its class' {@code ITEM_TYPE} field
+ * @param property the property we're aggregating on, i.e. for each possible value of this property, we are counting how many items of the specified type have that value
+ * @param query the {@link AggregateQuery} specifying the aggregation that should be perfomed
+ * @return a Map associating a specific value of the property to the cardinality of items with that value
+ * @see Item Item for a discussion of {@code ITEM_TYPE}
+ */
+ Map<String, Long> getAggregateOptimized(String itemType, String property, AggregateQuery query);
+
+ /**
* Retrieves the number of items of the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and matching the specified {@link Condition}.
*
* @param condition the condition the items must satisfy
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7e5c2d2/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
----------------------------------------------------------------------
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index d5fa185..c7ec8d0 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -1483,6 +1483,16 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
public Map<String, Long> aggregateQuery(final Condition filter, final BaseAggregate aggregate, final String itemType) {
+ return aggregateQuery(filter, aggregate, itemType, false);
+ }
+
+ @Override
+ public Map<String, Long> aggregateQueryOptimized(Condition filter, BaseAggregate aggregate, String itemType) {
+ return aggregateQuery(filter, aggregate, itemType, true);
+ }
+
+ private Map<String, Long> aggregateQuery(final Condition filter, final BaseAggregate aggregate, final String itemType, final boolean
+ optimized) {
return new InClassLoaderExecute<Map<String, Long>>(metricsService, this.getClass().getName() + ".aggregateQuery") {
@Override
@@ -1560,34 +1570,50 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
- if (filter != null) {
- AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", conditionESQueryBuilderDispatcher.buildFilter(filter));
+ // If the request is optimized then we don't need a global aggregation which is very slow and we can put the query with a
+ // filter on range items in the query block so we don't retrieve all the document before filtering the whole
+ if (optimized) {
for (AggregationBuilder aggregationBuilder : lastAggregation) {
- filterAggregation.subAggregation(aggregationBuilder);
+ builder.addAggregation(aggregationBuilder);
}
- lastAggregation = Collections.singletonList(filterAggregation);
- }
+ if (filter != null) {
+ builder.setQuery(conditionESQueryBuilderDispatcher.buildFilter(filter));
+ }
+ } else {
+ if (filter != null) {
+ AggregationBuilder filterAggregation = AggregationBuilders.filter("filter", conditionESQueryBuilderDispatcher.buildFilter(filter));
+ for (AggregationBuilder aggregationBuilder : lastAggregation) {
+ filterAggregation.subAggregation(aggregationBuilder);
+ }
+ lastAggregation = Collections.singletonList(filterAggregation);
+ }
- AggregationBuilder globalAggregation = AggregationBuilders.global("global");
- for (AggregationBuilder aggregationBuilder : lastAggregation) {
- globalAggregation.subAggregation(aggregationBuilder);
- }
+ AggregationBuilder globalAggregation = AggregationBuilders.global("global");
+ for (AggregationBuilder aggregationBuilder : lastAggregation) {
+ globalAggregation.subAggregation(aggregationBuilder);
+ }
- builder.addAggregation(globalAggregation);
+ builder.addAggregation(globalAggregation);
+ }
SearchResponse response = builder.execute().actionGet();
-
Aggregations aggregations = response.getAggregations();
if (aggregations != null) {
- Global globalAgg = aggregations.get("global");
- results.put("_all", globalAgg.getDocCount());
- aggregations = globalAgg.getAggregations();
-
- if (aggregations.get("filter") != null) {
- Filter filterAgg = aggregations.get("filter");
- results.put("_filtered", filterAgg.getDocCount());
- aggregations = filterAgg.getAggregations();
+ if (optimized) {
+ if (response.getHits() != null) {
+ results.put("_filtered", response.getHits().getTotalHits());
+ }
+ } else {
+ Global globalAgg = aggregations.get("global");
+ results.put("_all", globalAgg.getDocCount());
+ aggregations = globalAgg.getAggregations();
+
+ if (aggregations.get("filter") != null) {
+ Filter filterAgg = aggregations.get("filter");
+ results.put("_filtered", filterAgg.getDocCount());
+ aggregations = filterAgg.getAggregations();
+ }
}
if (aggregations.get("buckets") != null) {
MultiBucketsAggregation terms = aggregations.get("buckets");
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7e5c2d2/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
----------------------------------------------------------------------
diff --git a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
index b38531b..c98eb17 100644
--- a/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
+++ b/persistence-spi/src/main/java/org/apache/unomi/persistence/spi/PersistenceService.java
@@ -413,6 +413,7 @@ public interface PersistenceService {
/**
* Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} matching the optional specified condition and
* aggregated according to the specified {@link BaseAggregate}.
+ * Also return the global count of document matching the {@code ITEM_TYPE}
*
* @param filter the condition the items must match or {@code null} if no filtering is needed
* @param aggregate an aggregate specifying how matching items must be bundled
@@ -422,6 +423,18 @@ public interface PersistenceService {
Map<String, Long> aggregateQuery(Condition filter, BaseAggregate aggregate, String itemType);
/**
+ * Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} matching the optional specified condition and
+ * aggregated according to the specified {@link BaseAggregate}.
+ * This aggregate won't return the global count and should therefore be much faster than {@link #aggregateQuery(Condition, BaseAggregate, String)}
+ *
+ * @param filter the condition the items must match or {@code null} if no filtering is needed
+ * @param aggregate an aggregate specifying how matching items must be bundled
+ * @param itemType the String representation of the item type we want to retrieve the count of, as defined by its class' {@code ITEM_TYPE} field
+ * @return a Map associating aggregation dimension name as key and cardinality for that dimension as value
+ */
+ Map<String, Long> aggregateQueryOptimized(Condition filter, BaseAggregate aggregate, String itemType);
+
+ /**
* Updates the persistence's engine indices if needed.
*/
void refresh();
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7e5c2d2/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
----------------------------------------------------------------------
diff --git a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
index 7f85a1a..8a7a381 100644
--- a/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
+++ b/plugins/baseplugin/src/main/java/org/apache/unomi/plugins/baseplugin/conditions/BooleanConditionESQueryBuilder.java
@@ -54,14 +54,22 @@ public class BooleanConditionESQueryBuilder implements ConditionESQueryBuilder {
if (isAndOperator) {
QueryBuilder andFilter = dispatcher.buildFilter(conditions.get(i), context);
if (andFilter != null) {
- boolQueryBuilder.must(andFilter);
+ if (andFilter.getName().equals("range")) {
+ boolQueryBuilder.filter(andFilter);
+ } else {
+ boolQueryBuilder.must(andFilter);
+ }
} else {
logger.warn("Null filter for boolean AND sub condition " + conditions.get(i));
}
} else {
QueryBuilder orFilter = dispatcher.buildFilter(conditions.get(i), context);
if (orFilter != null) {
- boolQueryBuilder.should(orFilter);
+ if (orFilter.getName().equals("range")) {
+ boolQueryBuilder.filter(orFilter);
+ } else {
+ boolQueryBuilder.should(orFilter);
+ }
} else {
logger.warn("Null filter for boolean OR sub condition " + conditions.get(i));
}
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7e5c2d2/rest/src/main/java/org/apache/unomi/rest/QueryServiceEndPoint.java
----------------------------------------------------------------------
diff --git a/rest/src/main/java/org/apache/unomi/rest/QueryServiceEndPoint.java b/rest/src/main/java/org/apache/unomi/rest/QueryServiceEndPoint.java
index 33c6bdb..3d68e05 100644
--- a/rest/src/main/java/org/apache/unomi/rest/QueryServiceEndPoint.java
+++ b/rest/src/main/java/org/apache/unomi/rest/QueryServiceEndPoint.java
@@ -82,6 +82,7 @@ public class QueryServiceEndPoint {
*
* Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and aggregated by possible values of the specified
* property or, if the specified query is not {@code null}, perform that aggregate query.
+ * Also return the global count of document matching the {@code ITEM_TYPE}
*
* @param type the String representation of the item type we want to retrieve the count of, as defined by its class' {@code ITEM_TYPE} field
* @param property the property we're aggregating on, i.e. for each possible value of this property, we are counting how many items of the specified type have that value
@@ -96,6 +97,24 @@ public class QueryServiceEndPoint {
}
/**
+ * Retrieves the number of items with the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and aggregated by possible values of the specified
+ * property or, if the specified query is not {@code null}, perform that aggregate query.
+ * This aggregate won't return the global count and should therefore be much faster than {@link #getAggregate(String, String, AggregateQuery)}
+ *
+ * @param type the String representation of the item type we want to retrieve the count of, as defined by its class' {@code ITEM_TYPE} field
+ * @param property the property we're aggregating on, i.e. for each possible value of this property, we are counting how many items of the specified type have that value
+ * @param aggregateQuery the {@link AggregateQuery} specifying the aggregation that should be perfomed
+ * @return a Map associating a specific value of the property to the cardinality of items with that value
+ * @see Item Item for a discussion of {@code ITEM_TYPE}
+ */
+ @POST
+ @Path("/{type}/{property}/optimized")
+ public Map<String, Long> getAggregateOptimized(@PathParam("type") String type, @PathParam("property") String property, AggregateQuery
+ aggregateQuery) {
+ return queryService.getAggregateOptimized(type, property, aggregateQuery);
+ }
+
+ /**
* Retrieves the specified metrics for the specified field of items of the specified type as defined by the Item subclass public field {@code ITEM_TYPE} and matching the
* specified {@link Condition}.
*
http://git-wip-us.apache.org/repos/asf/incubator-unomi/blob/d7e5c2d2/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
----------------------------------------------------------------------
diff --git a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
index bb9de1d..5da7076 100644
--- a/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/services/QueryServiceImpl.java
@@ -58,6 +58,31 @@ public class QueryServiceImpl implements QueryService {
@Override
public Map<String, Long> getAggregate(String itemType, String property, AggregateQuery query) {
+ return getAggregate(itemType, property, query, false);
+ }
+
+ @Override
+ public Map<String, Long> getAggregateOptimized(String itemType, String property, AggregateQuery query) {
+ return getAggregate(itemType, property, query, true);
+ }
+
+ @Override
+ public Map<String, Double> getMetric(String type, String property, String slashConcatenatedMetrics, Condition condition) {
+ if (condition.getConditionType() == null) {
+ ParserHelper.resolveConditionType(definitionsService, condition);
+ }
+ return persistenceService.getSingleValuesMetrics(condition, slashConcatenatedMetrics.split("/"), property, type);
+ }
+
+ @Override
+ public long getQueryCount(String itemType, Condition condition) {
+ if (condition.getConditionType() == null) {
+ ParserHelper.resolveConditionType(definitionsService, condition);
+ }
+ return persistenceService.queryCount(condition, itemType);
+ }
+
+ private Map<String, Long> getAggregate(String itemType, String property, AggregateQuery query, boolean optimized) {
if (query != null) {
// resolve condition
if (query.getCondition() != null) {
@@ -65,6 +90,7 @@ public class QueryServiceImpl implements QueryService {
}
// resolve aggregate
+ BaseAggregate baseAggregate = null;
if (query.getAggregate() != null) {
String aggregateType = query.getAggregate().getType();
if (aggregateType != null) {
@@ -72,40 +98,31 @@ public class QueryServiceImpl implements QueryService {
if (aggregateType.equals("date")) {
String interval = (String) query.getAggregate().getParameters().get("interval");
String format = (String) query.getAggregate().getParameters().get("format");
- return persistenceService.aggregateQuery(query.getCondition(), new DateAggregate(property, interval, format), itemType);
+ baseAggregate = new DateAggregate(property, interval, format);
} else if (aggregateType.equals("dateRange") && query.getAggregate().getDateRanges() != null && query.getAggregate().getDateRanges().size() > 0) {
String format = (String) query.getAggregate().getParameters().get("format");
- return persistenceService.aggregateQuery(query.getCondition(), new DateRangeAggregate(query.getAggregate().getProperty(), format, query.getAggregate().getDateRanges()), itemType);
+ baseAggregate = new DateRangeAggregate(query.getAggregate().getProperty(), format, query.getAggregate().getDateRanges());
} else if (aggregateType.equals("numericRange") && query.getAggregate().getNumericRanges() != null && query.getAggregate().getNumericRanges().size() > 0) {
- return persistenceService.aggregateQuery(query.getCondition(), new NumericRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().getNumericRanges()), itemType);
+ baseAggregate = new NumericRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().getNumericRanges());
} else if (aggregateType.equals("ipRange") && query.getAggregate().ipRanges() != null && query.getAggregate().ipRanges().size() > 0) {
- return persistenceService.aggregateQuery(query.getCondition(), new IpRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().ipRanges()), itemType);
+ baseAggregate = new IpRangeAggregate(query.getAggregate().getProperty(), query.getAggregate().ipRanges());
}
}
}
+ if (baseAggregate == null) {
+ baseAggregate = new TermsAggregate(property);
+ }
+
// fall back on terms aggregate
- return persistenceService.aggregateQuery(query.getCondition(), new TermsAggregate(property), itemType);
+ if (optimized) {
+ return persistenceService.aggregateQueryOptimized(query.getCondition(), baseAggregate, itemType);
+ } else {
+ return persistenceService.aggregateQuery(query.getCondition(), baseAggregate, itemType);
+ }
}
return getAggregate(itemType, property);
}
- @Override
- public Map<String, Double> getMetric(String type, String property, String slashConcatenatedMetrics, Condition condition) {
- if (condition.getConditionType() == null) {
- ParserHelper.resolveConditionType(definitionsService, condition);
- }
- return persistenceService.getSingleValuesMetrics(condition, slashConcatenatedMetrics.split("/"), property, type);
- }
-
- @Override
- public long getQueryCount(String itemType, Condition condition) {
- if (condition.getConditionType() == null) {
- ParserHelper.resolveConditionType(definitionsService, condition);
- }
- return persistenceService.queryCount(condition, itemType);
- }
-
-
}
\ No newline at end of file