You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2019/12/03 16:35:43 UTC
[unomi] 01/01: UNOMI-225 ElasticSearch 7 fix for changes in the
total hits - Replaced total hits with count API requests when possible -
For aggregations replaced with a manual sum of all buckets - Added in the
PartialList the relation for the total hits so that users of the API can
know if the total size is EQUAL or GREATER_THAN_OR_EQUAL_TO
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch UNOMI-225-es7-fix-counts
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 9dc21cf16e03d72a9f9b734f12ee8198b7b1e62f
Author: Serge Huber <sh...@apache.org>
AuthorDate: Tue Dec 3 17:35:35 2019 +0100
UNOMI-225 ElasticSearch 7 fix for changes in the total hits
- Replaced total hits with count API requests when possible
- For aggregations replaced with a manual sum of all buckets
- Added in the PartialList the relation for the total hits so that users of the API can know if the total size is EQUAL or GREATER_THAN_OR_EQUAL_TO
Signed-off-by: Serge Huber <sh...@apache.org>
---
.../java/org/apache/unomi/api/PartialList.java | 28 +++++++++++++++++++++-
.../apache/unomi/services/UserListServiceImpl.java | 4 ++--
.../ElasticSearchPersistenceServiceImpl.java | 28 ++++++++++++++++++++--
.../unomi/services/impl/AbstractServiceImpl.java | 4 ++--
.../services/impl/goals/GoalsServiceImpl.java | 2 +-
.../services/impl/rules/RulesServiceImpl.java | 4 ++--
.../services/impl/segments/SegmentServiceImpl.java | 2 +-
7 files changed, 61 insertions(+), 11 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/PartialList.java b/api/src/main/java/org/apache/unomi/api/PartialList.java
index a78eb73..2128f93 100644
--- a/api/src/main/java/org/apache/unomi/api/PartialList.java
+++ b/api/src/main/java/org/apache/unomi/api/PartialList.java
@@ -35,10 +35,21 @@ public class PartialList<T> implements Serializable {
private long offset;
private long pageSize;
private long totalSize;
+ private Relation totalSizeRelation;
private String scrollIdentifier = null;
private String scrollTimeValidity = null;
/**
+ * This enum exists to replicate Lucene's total hits relation in a back-end agnostic way. Basically Lucene will
+ * by default not report accurate total hit counts above a certain threshold for performance reasons. Using the
+ * relation we can understand if we are in the case of an accurate hit or not.
+ */
+ public enum Relation {
+ EQUAL,
+ GREATER_THAN_OR_EQUAL_TO
+ }
+
+ /**
* Instantiates a new PartialList.
*/
public PartialList() {
@@ -46,6 +57,7 @@ public class PartialList<T> implements Serializable {
offset = 0;
pageSize = 0;
totalSize = 0;
+ totalSizeRelation = Relation.EQUAL;
}
/**
@@ -56,11 +68,12 @@ public class PartialList<T> implements Serializable {
* @param pageSize the number of elements this PartialList contains
* @param totalSize the total size of elements in the original List
*/
- public PartialList(List<T> list, long offset, long pageSize, long totalSize) {
+ public PartialList(List<T> list, long offset, long pageSize, long totalSize, Relation totalSizeRelation) {
this.list = list;
this.offset = offset;
this.pageSize = pageSize;
this.totalSize = totalSize;
+ this.totalSizeRelation = totalSizeRelation;
}
/**
@@ -164,4 +177,17 @@ public class PartialList<T> implements Serializable {
public void setScrollTimeValidity(String scrollTimeValidity) {
this.scrollTimeValidity = scrollTimeValidity;
}
+
+ /**
+ * Retrieve the relation to the total site, wether it is equal to or greater than the value stored in the
+ * totalSize property.
+ * @return a Relation enum value that describes the type of total size we have in this object.
+ */
+ public Relation getTotalSizeRelation() {
+ return totalSizeRelation;
+ }
+
+ public void setTotalSizeRelation(Relation totalSizeRelation) {
+ this.totalSizeRelation = totalSizeRelation;
+ }
}
diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
index e60cb19..237a52a 100644
--- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
+++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
@@ -51,7 +51,7 @@ public class UserListServiceImpl implements UserListService {
for (UserList definition : userLists.getList()) {
metadata.add(definition.getMetadata());
}
- return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize());
+ return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize(), userLists.getTotalSizeRelation());
}
public PartialList<Metadata> getListMetadatas(Query query) {
@@ -64,7 +64,7 @@ public class UserListServiceImpl implements UserListService {
for (UserList definition : userLists.getList()) {
metadata.add(definition.getMetadata());
}
- return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize());
+ return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize(), userLists.getTotalSizeRelation());
}
@Override
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index de47836..568943b 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -24,6 +24,7 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.lucene.search.TotalHits;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.PartialList;
import org.apache.unomi.api.TimestampedItem;
@@ -53,6 +54,8 @@ import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.*;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.core.MainResponse;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -1321,6 +1324,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
protected Long execute(Object... args) throws IOException {
+
+ CountRequest countRequest = new CountRequest(getIndexNameForQuery(itemType));
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(filter);
+ countRequest.source(searchSourceBuilder);
+ CountResponse response = client.count(countRequest, RequestOptions.DEFAULT);
+ return response.getCount();
+ /*
SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(filter);
@@ -1328,6 +1339,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
return response.getHits().getTotalHits().value;
+ */
}
}.catchingExecuteInClassLoader(true);
}
@@ -1340,6 +1352,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
List<T> results = new ArrayList<T>();
String scrollIdentifier = null;
long totalHits = 0;
+ PartialList.Relation totalHitsRelation = PartialList.Relation.EQUAL;
try {
String itemType = Item.getItemType(clazz);
TimeValue keepAlive = TimeValue.timeValueHours(1);
@@ -1423,6 +1436,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchHits searchHits = response.getHits();
scrollIdentifier = response.getScrollId();
totalHits = searchHits.getTotalHits().value;
+ totalHitsRelation = getTotalHitsRelation(searchHits.getTotalHits());
for (SearchHit searchHit : searchHits) {
String sourceAsString = searchHit.getSourceAsString();
final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
@@ -1435,7 +1449,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
throw new Exception("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
}
- PartialList<T> result = new PartialList<T>(results, offset, size, totalHits);
+ PartialList<T> result = new PartialList<T>(results, offset, size, totalHits, totalHitsRelation);
if (scrollIdentifier != null && totalHits != 0) {
result.setScrollIdentifier(scrollIdentifier);
result.setScrollTimeValidity(scrollTimeValidity);
@@ -1445,6 +1459,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}.catchingExecuteInClassLoader(true);
}
+ private PartialList.Relation getTotalHitsRelation(TotalHits totalHits) {
+ return TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(totalHits.relation) ? PartialList.Relation.GREATER_THAN_OR_EQUAL_TO : PartialList.Relation.EQUAL;
+ }
+
@Override
public <T extends Item> PartialList<T> continueScrollQuery(final Class<T> clazz, final String scrollIdentifier, final String scrollTimeValidity) {
return new InClassLoaderExecute<PartialList<T>>(metricsService, this.getClass().getName() + ".continueScrollQuery") {
@@ -1474,7 +1492,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
results.add(value);
}
}
- PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value);
+ PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value, getTotalHitsRelation(response.getHits().getTotalHits()));
if (scrollIdentifier != null) {
result.setScrollIdentifier(scrollIdentifier);
result.setScrollTimeValidity(scrollTimeValidity);
@@ -1632,13 +1650,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
if (aggregations.get("buckets") != null) {
+ long totalDocCount = 0;
MultiBucketsAggregation terms = aggregations.get("buckets");
for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
results.put(bucket.getKeyAsString(), bucket.getDocCount());
+ totalDocCount += bucket.getDocCount();
}
SingleBucketAggregation missing = aggregations.get("missing");
if (missing.getDocCount() > 0) {
results.put("_missing", missing.getDocCount());
+ totalDocCount += missing.getDocCount();
+ }
+ if (response.getHits() != null && TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(response.getHits().getTotalHits().relation)) {
+ results.put("_filtered", totalDocCount);
}
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
index 22cf911..1368105 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
@@ -50,7 +50,7 @@ public abstract class AbstractServiceImpl {
for (T definition : items.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize());
+ return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize(), items.getTotalSizeRelation());
}
protected <T extends MetadataItem> PartialList<Metadata> getMetadatas(Query query, Class<T> clazz) {
@@ -63,6 +63,6 @@ public abstract class AbstractServiceImpl {
for (T definition : items.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize());
+ return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize(), items.getTotalSizeRelation());
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
index 336eac9..cf2788e 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
@@ -360,7 +360,7 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener
details.add(campaignDetail);
}
}
- return new PartialList<>(details, campaigns.getOffset(), campaigns.getPageSize(), campaigns.getTotalSize());
+ return new PartialList<>(details, campaigns.getOffset(), campaigns.getPageSize(), campaigns.getTotalSize(), campaigns.getTotalSizeRelation());
}
public CampaignDetail getCampaignDetail(String id) {
diff --git a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
index 6506b42..4f7161a 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
@@ -314,7 +314,7 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
for (Rule definition : rules.getList()) {
descriptions.add(definition.getMetadata());
}
- return new PartialList<>(descriptions, rules.getOffset(), rules.getPageSize(), rules.getTotalSize());
+ return new PartialList<>(descriptions, rules.getOffset(), rules.getPageSize(), rules.getTotalSize(), rules.getTotalSizeRelation());
}
public PartialList<Rule> getRuleDetails(Query query) {
@@ -325,7 +325,7 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
PartialList<Rule> rules = persistenceService.query(query.getCondition(), query.getSortby(), Rule.class, query.getOffset(), query.getLimit());
List<Rule> details = new LinkedList<>();
details.addAll(rules.getList());
- return new PartialList<>(details, rules.getOffset(), rules.getPageSize(), rules.getTotalSize());
+ return new PartialList<>(details, rules.getOffset(), rules.getPageSize(), rules.getTotalSize(), rules.getTotalSizeRelation());
}
public Rule getRule(String ruleId) {
diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
index 047751c..212c1b1 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
@@ -198,7 +198,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
for (Segment definition : segments.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, segments.getOffset(), segments.getPageSize(), segments.getTotalSize());
+ return new PartialList<>(details, segments.getOffset(), segments.getPageSize(), segments.getTotalSize(), segments.getTotalSizeRelation());
}
public PartialList<Metadata> getSegmentMetadatas(Query query) {