You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2022/06/16 13:34:00 UTC
[jackrabbit-oak] 04/32: ElasticResultRowAsyncIterator should not limit result set to 10k results (#588)
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-9780
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit b7cb2c870b90343273697e513af4ab4e4d2f16b7
Author: Fabrizio Fortino <fa...@gmail.com>
AuthorDate: Wed Jun 8 17:49:10 2022 +0200
ElasticResultRowAsyncIterator should not limit result set to 10k results (#588)
* fix: ElasticResultRowAsyncIterator should not limit result set to 10k results
* minor: improve readability
* TRACK_TOTAL_HITS_DEFAULT back to 10k
---
.../index/elastic/ElasticIndexDefinition.java | 5 ++++
.../query/async/ElasticResultRowAsyncIterator.java | 9 ++++++-
.../plugins/index/elastic/ElasticContentTest.java | 29 ++++++++++++++++++++++
3 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
index e12797ae8a..9fb5c0d06d 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
@@ -63,6 +63,9 @@ public class ElasticIndexDefinition extends IndexDefinition {
public static final String QUERY_FETCH_SIZES = "queryFetchSizes";
public static final Long[] QUERY_FETCH_SIZES_DEFAULT = new Long[]{100L, 1000L};
+ public static final String TRACK_TOTAL_HITS = "trackTotalHits";
+ public static final Integer TRACK_TOTAL_HITS_DEFAULT = 10000;
+
/**
* Hidden property for storing a seed value to be used as suffix in remote index name.
*/
@@ -114,6 +117,7 @@ public class ElasticIndexDefinition extends IndexDefinition {
public final int numberOfShards;
public final int numberOfReplicas;
public final int[] queryFetchSizes;
+ public final Integer trackTotalHits;
private final Map<String, List<PropertyDefinition>> propertiesByName;
private final List<PropertyDefinition> dynamicBoostProperties;
@@ -134,6 +138,7 @@ public class ElasticIndexDefinition extends IndexDefinition {
this.similarityTagsBoost = getOptionalValue(defn, SIMILARITY_TAGS_BOOST, SIMILARITY_TAGS_BOOST_DEFAULT);
this.queryFetchSizes = Arrays.stream(getOptionalValues(defn, QUERY_FETCH_SIZES, Type.LONGS, Long.class, QUERY_FETCH_SIZES_DEFAULT))
.mapToInt(Long::intValue).toArray();
+ this.trackTotalHits = getOptionalValue(defn, TRACK_TOTAL_HITS, TRACK_TOTAL_HITS_DEFAULT);
this.propertiesByName = getDefinedRules()
.stream()
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/ElasticResultRowAsyncIterator.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/ElasticResultRowAsyncIterator.java
index d53038e036..e109a772b9 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/ElasticResultRowAsyncIterator.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/ElasticResultRowAsyncIterator.java
@@ -22,6 +22,7 @@ import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.core.search.SourceConfig;
+import co.elastic.clients.elasticsearch.core.search.TotalHitsRelation;
import com.fasterxml.jackson.databind.node.ObjectNode;
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticMetricHandler;
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexNode;
@@ -220,6 +221,7 @@ public class ElasticResultRowAsyncIterator implements Iterator<FulltextResultRow
SearchRequest searchReq = SearchRequest.of(builder -> {
builder
.index(indexNode.getDefinition().getIndexAlias())
+ .trackTotalHits(thb -> thb.count(indexNode.getDefinition().trackTotalHits))
.sort(sorts)
.source(sourceConfig)
.query(query)
@@ -272,7 +274,11 @@ public class ElasticResultRowAsyncIterator implements Iterator<FulltextResultRow
LOG.debug("Processing search response that took {} to read {}/{} docs", searchResponse.took(), hitsSize, totalHits);
lastHitSortValues = searchHits.get(hitsSize - 1).sort();
scannedRows += hitsSize;
- anyDataLeft.set(totalHits > scannedRows);
+ if (searchResponse.hits().total().relation() == TotalHitsRelation.Eq) {
+ anyDataLeft.set(totalHits > scannedRows);
+ } else {
+ anyDataLeft.set(true);
+ }
estimator.update(indexPlan.getFilter(), totalHits);
// now that we got the last hit we can release the semaphore to potentially unlock other requests
@@ -325,6 +331,7 @@ public class ElasticResultRowAsyncIterator implements Iterator<FulltextResultRow
if (semaphore.tryAcquire() && anyDataLeft.get()) {
final SearchRequest searchReq = SearchRequest.of(s -> s
.index(indexNode.getDefinition().getIndexAlias())
+ .trackTotalHits(thb -> thb.count(indexNode.getDefinition().trackTotalHits))
.sort(sorts)
.source(sourceConfig)
.searchAfter(lastHitSortValues)
diff --git a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticContentTest.java b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticContentTest.java
index 2b12868b5f..e915744e7f 100644
--- a/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticContentTest.java
+++ b/oak-search-elastic/src/test/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticContentTest.java
@@ -29,6 +29,7 @@ import java.util.List;
import java.util.Random;
import java.util.UUID;
import java.util.function.BiConsumer;
+import java.util.stream.Collectors;
import java.util.stream.IntStream;
import static org.apache.jackrabbit.oak.plugins.index.elastic.ElasticTestUtils.randomString;
@@ -208,4 +209,32 @@ public class ElasticContentTest extends ElasticAbstractQueryTest {
assertQuery("select [jcr:path] from [nt:base] where [c] = 'text'", results);
verify(spyMetricHandler, times(1)).markQuery(anyString(), anyBoolean());
}
+
+ @Test
+ public void indexWithLowTrackTotalHits() throws Exception {
+ BiConsumer<String, Iterable<Long>> buildIndex = (p, fetchSizes) -> {
+ IndexDefinitionBuilder builder = createIndex(p).noAsync();
+ builder.getBuilderTree().setProperty("queryFetchSizes", fetchSizes, Type.LONGS);
+ builder.getBuilderTree().setProperty("trackTotalHits", 10L, Type.LONG);
+ builder.indexRule("nt:base").property(p).propertyIndex();
+ setIndex(p + "_" + UUID.randomUUID(), builder);
+ };
+
+ buildIndex.accept("a", Collections.singletonList(10L));
+ root.commit();
+
+ Tree content = root.getTree("/").addChild("content");
+
+ List<String> results = IntStream.range(0, 100)
+ .mapToObj(n -> {
+ Tree child = content.addChild("child_" + n);
+ child.setProperty("a", "text");
+ return "/content/child_" + n;
+ })
+ .collect(Collectors.toList());
+
+ root.commit();
+
+ assertEventually(() -> assertQuery("select [jcr:path] from [nt:base] where [a] = 'text'", results));
+ }
}