You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@unomi.apache.org by sh...@apache.org on 2019/12/03 16:35:42 UTC
[unomi] branch UNOMI-225-es7-fix-counts created (now 9dc21cf)
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a change to branch UNOMI-225-es7-fix-counts
in repository https://gitbox.apache.org/repos/asf/unomi.git.
at 9dc21cf UNOMI-225 ElasticSearch 7 fix for changes in the total hits - Replaced total hits with count API requests when possible - For aggregations replaced with a manual sum of all buckets - Added in the PartialList the relation for the total hits so that users of the API can know if the total size is EQUAL or GREATER_THAN_OR_EQUAL_TO
This branch includes the following new commits:
new 9dc21cf UNOMI-225 ElasticSearch 7 fix for changes in the total hits - Replaced total hits with count API requests when possible - For aggregations replaced with a manual sum of all buckets - Added in the PartialList the relation for the total hits so that users of the API can know if the total size is EQUAL or GREATER_THAN_OR_EQUAL_TO
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[unomi] 01/01: UNOMI-225 ElasticSearch 7 fix for changes in the
total hits - Replaced total hits with count API requests when possible -
For aggregations replaced with a manual sum of all buckets - Added in the
PartialList the relation for the total hits so that users of the API can
know if the total size is EQUAL or GREATER_THAN_OR_EQUAL_TO
Posted by sh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
shuber pushed a commit to branch UNOMI-225-es7-fix-counts
in repository https://gitbox.apache.org/repos/asf/unomi.git
commit 9dc21cf16e03d72a9f9b734f12ee8198b7b1e62f
Author: Serge Huber <sh...@apache.org>
AuthorDate: Tue Dec 3 17:35:35 2019 +0100
UNOMI-225 ElasticSearch 7 fix for changes in the total hits
- Replaced total hits with count API requests when possible
- For aggregations replaced with a manual sum of all buckets
- Added in the PartialList the relation for the total hits so that users of the API can know if the total size is EQUAL or GREATER_THAN_OR_EQUAL_TO
Signed-off-by: Serge Huber <sh...@apache.org>
---
.../java/org/apache/unomi/api/PartialList.java | 28 +++++++++++++++++++++-
.../apache/unomi/services/UserListServiceImpl.java | 4 ++--
.../ElasticSearchPersistenceServiceImpl.java | 28 ++++++++++++++++++++--
.../unomi/services/impl/AbstractServiceImpl.java | 4 ++--
.../services/impl/goals/GoalsServiceImpl.java | 2 +-
.../services/impl/rules/RulesServiceImpl.java | 4 ++--
.../services/impl/segments/SegmentServiceImpl.java | 2 +-
7 files changed, 61 insertions(+), 11 deletions(-)
diff --git a/api/src/main/java/org/apache/unomi/api/PartialList.java b/api/src/main/java/org/apache/unomi/api/PartialList.java
index a78eb73..2128f93 100644
--- a/api/src/main/java/org/apache/unomi/api/PartialList.java
+++ b/api/src/main/java/org/apache/unomi/api/PartialList.java
@@ -35,10 +35,21 @@ public class PartialList<T> implements Serializable {
private long offset;
private long pageSize;
private long totalSize;
+ private Relation totalSizeRelation;
private String scrollIdentifier = null;
private String scrollTimeValidity = null;
/**
+ * This enum exists to replicate Lucene's total hits relation in a back-end agnostic way. Basically Lucene will
+ * by default not report accurate total hit counts above a certain threshold for performance reasons. Using the
+ * relation we can understand if we are in the case of an accurate hit or not.
+ */
+ public enum Relation {
+ EQUAL,
+ GREATER_THAN_OR_EQUAL_TO
+ }
+
+ /**
* Instantiates a new PartialList.
*/
public PartialList() {
@@ -46,6 +57,7 @@ public class PartialList<T> implements Serializable {
offset = 0;
pageSize = 0;
totalSize = 0;
+ totalSizeRelation = Relation.EQUAL;
}
/**
@@ -56,11 +68,12 @@ public class PartialList<T> implements Serializable {
* @param pageSize the number of elements this PartialList contains
* @param totalSize the total size of elements in the original List
*/
- public PartialList(List<T> list, long offset, long pageSize, long totalSize) {
+ public PartialList(List<T> list, long offset, long pageSize, long totalSize, Relation totalSizeRelation) {
this.list = list;
this.offset = offset;
this.pageSize = pageSize;
this.totalSize = totalSize;
+ this.totalSizeRelation = totalSizeRelation;
}
/**
@@ -164,4 +177,17 @@ public class PartialList<T> implements Serializable {
public void setScrollTimeValidity(String scrollTimeValidity) {
this.scrollTimeValidity = scrollTimeValidity;
}
+
+ /**
+ * Retrieve the relation to the total site, wether it is equal to or greater than the value stored in the
+ * totalSize property.
+ * @return a Relation enum value that describes the type of total size we have in this object.
+ */
+ public Relation getTotalSizeRelation() {
+ return totalSizeRelation;
+ }
+
+ public void setTotalSizeRelation(Relation totalSizeRelation) {
+ this.totalSizeRelation = totalSizeRelation;
+ }
}
diff --git a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
index e60cb19..237a52a 100644
--- a/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
+++ b/extensions/lists-extension/services/src/main/java/org/apache/unomi/services/UserListServiceImpl.java
@@ -51,7 +51,7 @@ public class UserListServiceImpl implements UserListService {
for (UserList definition : userLists.getList()) {
metadata.add(definition.getMetadata());
}
- return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize());
+ return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize(), userLists.getTotalSizeRelation());
}
public PartialList<Metadata> getListMetadatas(Query query) {
@@ -64,7 +64,7 @@ public class UserListServiceImpl implements UserListService {
for (UserList definition : userLists.getList()) {
metadata.add(definition.getMetadata());
}
- return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize());
+ return new PartialList<>(metadata, userLists.getOffset(), userLists.getPageSize(), userLists.getTotalSize(), userLists.getTotalSizeRelation());
}
@Override
diff --git a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
index de47836..568943b 100644
--- a/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
+++ b/persistence-elasticsearch/core/src/main/java/org/apache/unomi/persistence/elasticsearch/ElasticSearchPersistenceServiceImpl.java
@@ -24,6 +24,7 @@ import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
+import org.apache.lucene.search.TotalHits;
import org.apache.unomi.api.Item;
import org.apache.unomi.api.PartialList;
import org.apache.unomi.api.TimestampedItem;
@@ -53,6 +54,8 @@ import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.*;
+import org.elasticsearch.client.core.CountRequest;
+import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.client.core.MainResponse;
import org.elasticsearch.client.indices.*;
import org.elasticsearch.cluster.metadata.MappingMetaData;
@@ -1321,6 +1324,14 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
@Override
protected Long execute(Object... args) throws IOException {
+
+ CountRequest countRequest = new CountRequest(getIndexNameForQuery(itemType));
+ SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+ searchSourceBuilder.query(filter);
+ countRequest.source(searchSourceBuilder);
+ CountResponse response = client.count(countRequest, RequestOptions.DEFAULT);
+ return response.getCount();
+ /*
SearchRequest searchRequest = new SearchRequest(getIndexNameForQuery(itemType));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(filter);
@@ -1328,6 +1339,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, RequestOptions.DEFAULT);
return response.getHits().getTotalHits().value;
+ */
}
}.catchingExecuteInClassLoader(true);
}
@@ -1340,6 +1352,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
List<T> results = new ArrayList<T>();
String scrollIdentifier = null;
long totalHits = 0;
+ PartialList.Relation totalHitsRelation = PartialList.Relation.EQUAL;
try {
String itemType = Item.getItemType(clazz);
TimeValue keepAlive = TimeValue.timeValueHours(1);
@@ -1423,6 +1436,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
SearchHits searchHits = response.getHits();
scrollIdentifier = response.getScrollId();
totalHits = searchHits.getTotalHits().value;
+ totalHitsRelation = getTotalHitsRelation(searchHits.getTotalHits());
for (SearchHit searchHit : searchHits) {
String sourceAsString = searchHit.getSourceAsString();
final T value = ESCustomObjectMapper.getObjectMapper().readValue(sourceAsString, clazz);
@@ -1435,7 +1449,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
throw new Exception("Error loading itemType=" + clazz.getName() + " query=" + query + " sortBy=" + sortBy, t);
}
- PartialList<T> result = new PartialList<T>(results, offset, size, totalHits);
+ PartialList<T> result = new PartialList<T>(results, offset, size, totalHits, totalHitsRelation);
if (scrollIdentifier != null && totalHits != 0) {
result.setScrollIdentifier(scrollIdentifier);
result.setScrollTimeValidity(scrollTimeValidity);
@@ -1445,6 +1459,10 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}.catchingExecuteInClassLoader(true);
}
+ private PartialList.Relation getTotalHitsRelation(TotalHits totalHits) {
+ return TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(totalHits.relation) ? PartialList.Relation.GREATER_THAN_OR_EQUAL_TO : PartialList.Relation.EQUAL;
+ }
+
@Override
public <T extends Item> PartialList<T> continueScrollQuery(final Class<T> clazz, final String scrollIdentifier, final String scrollTimeValidity) {
return new InClassLoaderExecute<PartialList<T>>(metricsService, this.getClass().getName() + ".continueScrollQuery") {
@@ -1474,7 +1492,7 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
results.add(value);
}
}
- PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value);
+ PartialList<T> result = new PartialList<T>(results, 0, response.getHits().getHits().length, response.getHits().getTotalHits().value, getTotalHitsRelation(response.getHits().getTotalHits()));
if (scrollIdentifier != null) {
result.setScrollIdentifier(scrollIdentifier);
result.setScrollTimeValidity(scrollTimeValidity);
@@ -1632,13 +1650,19 @@ public class ElasticSearchPersistenceServiceImpl implements PersistenceService,
}
}
if (aggregations.get("buckets") != null) {
+ long totalDocCount = 0;
MultiBucketsAggregation terms = aggregations.get("buckets");
for (MultiBucketsAggregation.Bucket bucket : terms.getBuckets()) {
results.put(bucket.getKeyAsString(), bucket.getDocCount());
+ totalDocCount += bucket.getDocCount();
}
SingleBucketAggregation missing = aggregations.get("missing");
if (missing.getDocCount() > 0) {
results.put("_missing", missing.getDocCount());
+ totalDocCount += missing.getDocCount();
+ }
+ if (response.getHits() != null && TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO.equals(response.getHits().getTotalHits().relation)) {
+ results.put("_filtered", totalDocCount);
}
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
index 22cf911..1368105 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/AbstractServiceImpl.java
@@ -50,7 +50,7 @@ public abstract class AbstractServiceImpl {
for (T definition : items.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize());
+ return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize(), items.getTotalSizeRelation());
}
protected <T extends MetadataItem> PartialList<Metadata> getMetadatas(Query query, Class<T> clazz) {
@@ -63,6 +63,6 @@ public abstract class AbstractServiceImpl {
for (T definition : items.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize());
+ return new PartialList<>(details, items.getOffset(), items.getPageSize(), items.getTotalSize(), items.getTotalSizeRelation());
}
}
diff --git a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
index 336eac9..cf2788e 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/goals/GoalsServiceImpl.java
@@ -360,7 +360,7 @@ public class GoalsServiceImpl implements GoalsService, SynchronousBundleListener
details.add(campaignDetail);
}
}
- return new PartialList<>(details, campaigns.getOffset(), campaigns.getPageSize(), campaigns.getTotalSize());
+ return new PartialList<>(details, campaigns.getOffset(), campaigns.getPageSize(), campaigns.getTotalSize(), campaigns.getTotalSizeRelation());
}
public CampaignDetail getCampaignDetail(String id) {
diff --git a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
index 6506b42..4f7161a 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/rules/RulesServiceImpl.java
@@ -314,7 +314,7 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
for (Rule definition : rules.getList()) {
descriptions.add(definition.getMetadata());
}
- return new PartialList<>(descriptions, rules.getOffset(), rules.getPageSize(), rules.getTotalSize());
+ return new PartialList<>(descriptions, rules.getOffset(), rules.getPageSize(), rules.getTotalSize(), rules.getTotalSizeRelation());
}
public PartialList<Rule> getRuleDetails(Query query) {
@@ -325,7 +325,7 @@ public class RulesServiceImpl implements RulesService, EventListenerService, Syn
PartialList<Rule> rules = persistenceService.query(query.getCondition(), query.getSortby(), Rule.class, query.getOffset(), query.getLimit());
List<Rule> details = new LinkedList<>();
details.addAll(rules.getList());
- return new PartialList<>(details, rules.getOffset(), rules.getPageSize(), rules.getTotalSize());
+ return new PartialList<>(details, rules.getOffset(), rules.getPageSize(), rules.getTotalSize(), rules.getTotalSizeRelation());
}
public Rule getRule(String ruleId) {
diff --git a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
index 047751c..212c1b1 100644
--- a/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
+++ b/services/src/main/java/org/apache/unomi/services/impl/segments/SegmentServiceImpl.java
@@ -198,7 +198,7 @@ public class SegmentServiceImpl extends AbstractServiceImpl implements SegmentSe
for (Segment definition : segments.getList()) {
details.add(definition.getMetadata());
}
- return new PartialList<>(details, segments.getOffset(), segments.getPageSize(), segments.getTotalSize());
+ return new PartialList<>(details, segments.getOffset(), segments.getPageSize(), segments.getTotalSize(), segments.getTotalSizeRelation());
}
public PartialList<Metadata> getSegmentMetadatas(Query query) {