You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fo...@apache.org on 2024/10/16 10:16:40 UTC
(jackrabbit-oak) branch trunk updated: OAK-11197: improve statistical facets by introducing pre-computed random values (#1791)
This is an automated email from the ASF dual-hosted git repository.
fortino pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 1122bd71af OAK-11197: improve statistical facets by introducing pre-computed random values (#1791)
1122bd71af is described below
commit 1122bd71afd49d21a680aecb3dd91b880dfb7cf3
Author: Fabrizio Fortino <fa...@gmail.com>
AuthorDate: Wed Oct 16 12:16:35 2024 +0200
OAK-11197: improve statistical facets by introducing pre-computed random values (#1791)
* OAK-11197: improve statistical facets by introducing pre-computed random values
* OAK-11197: use short for random values to reduce disk footprint
* OAK-11197: fix issue in ElasticStatisticalFacetAsyncProvider
* OAK-11197: (minor) fix indentation
* OAK-11197: replaced sha-256 random value generation with murmur3
* OAK-11197: use murmur3 impl from apache commons + seed from index def
---
.../plugins/index/elastic/ElasticIndexDefinition.java | 5 +++++
.../oak/plugins/index/elastic/index/ElasticDocument.java | 10 ++++++++++
.../index/elastic/index/ElasticDocumentMaker.java | 3 ++-
.../plugins/index/elastic/index/ElasticIndexHelper.java | 2 ++
.../facets/ElasticStatisticalFacetAsyncProvider.java | 16 +++++++++-------
.../oak/plugins/index/search/FulltextIndexConstants.java | 2 +-
6 files changed, 29 insertions(+), 9 deletions(-)
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
index 284e4d7e01..aa4079386a 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticIndexDefinition.java
@@ -99,6 +99,11 @@ public class ElasticIndexDefinition extends IndexDefinition {
*/
public static final String DYNAMIC_BOOST_FULLTEXT = ":dynamic-boost-ft";
+ /**
+ * Precomputed random value based on the path of the document
+ */
+ public static final String PATH_RANDOM_VALUE = ":path-random-value";
+
/**
* Dynamic properties are fields that are not explicitly defined in the index mapping and are added on the fly when a document is indexed.
* Examples: aggregations with relative nodes, regex properties (to be supported), etc.
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
index aa9d414786..ba89d0eefb 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocument.java
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugins.index.elastic.index;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
+import org.apache.commons.codec.digest.MurmurHash3;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
@@ -26,6 +27,7 @@ import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.spi.binary.BlobByteSource;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -40,6 +42,8 @@ public class ElasticDocument {
@JsonProperty(FieldNames.PATH)
public final String path;
+ @JsonProperty(ElasticIndexDefinition.PATH_RANDOM_VALUE)
+ public final int pathRandomValue;
@JsonProperty(FieldNames.FULLTEXT)
public final Set<String> fulltext;
@JsonProperty(FieldNames.SUGGEST)
@@ -57,7 +61,13 @@ public class ElasticDocument {
private final List<Map<String, Object>> dynamicProperties;
ElasticDocument(String path) {
+ this(path, 0);
+ }
+
+ ElasticDocument(String path, int seed) {
this.path = path;
+ byte[] pathBytes = path.getBytes(StandardCharsets.UTF_8);
+ this.pathRandomValue = MurmurHash3.hash32x86(pathBytes, 0, pathBytes.length, seed);
this.fulltext = new LinkedHashSet<>();
this.suggest = new LinkedHashSet<>();
this.spellcheck = new LinkedHashSet<>();
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
index 6f739a8682..6927388380 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticDocumentMaker.java
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugins.index.elastic.index;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.Aggregate;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
@@ -48,7 +49,7 @@ public class ElasticDocumentMaker extends FulltextDocumentMaker<ElasticDocument>
@Override
protected ElasticDocument initDoc() {
- return new ElasticDocument(path);
+ return new ElasticDocument(path, (int) definition.getDefinitionNodeState().getLong(ElasticIndexDefinition.PROP_INDEX_NAME_SEED));
}
@Override
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
index 20fea31202..116c617d54 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/index/ElasticIndexHelper.java
@@ -96,6 +96,8 @@ class ElasticIndexHelper {
private static void mapInternalProperties(@NotNull TypeMapping.Builder builder) {
builder.properties(FieldNames.PATH,
b1 -> b1.keyword(builder3 -> builder3))
+ .properties(ElasticIndexDefinition.PATH_RANDOM_VALUE,
+ b1 -> b1.integer(b2 -> b2.docValues(true).index(false)))
.properties(FieldNames.ANCESTORS,
b1 -> b1.text(
b2 -> b2.analyzer("ancestor_analyzer")
diff --git a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/facets/ElasticStatisticalFacetAsyncProvider.java b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/facets/ElasticStatisticalFacetAsyncProvider.java
index bc79385308..5eaf9b2a99 100644
--- a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/facets/ElasticStatisticalFacetAsyncProvider.java
+++ b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/query/async/facets/ElasticStatisticalFacetAsyncProvider.java
@@ -18,7 +18,7 @@ package org.apache.jackrabbit.oak.plugins.index.elastic.query.async.facets;
import co.elastic.clients.elasticsearch._types.aggregations.Aggregate;
import co.elastic.clients.elasticsearch._types.aggregations.StringTermsBucket;
-import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
+import co.elastic.clients.elasticsearch._types.mapping.FieldType;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
@@ -74,17 +74,19 @@ public class ElasticStatisticalFacetAsyncProvider implements ElasticFacetProvide
this.isAccessible = isAccessible;
this.facetFields = elasticRequestHandler.facetFields().collect(Collectors.toSet());
- BoolQuery.Builder builder = elasticRequestHandler.baseQueryBuilder();
- builder.should(sb -> sb.functionScore(fsb ->
- fsb.functions(f -> f.randomScore(rsb -> rsb.seed("" + randomSeed).field(FieldNames.PATH)))
- ));
-
SearchRequest searchRequest = SearchRequest.of(srb -> srb.index(indexDefinition.getIndexAlias())
.trackTotalHits(thb -> thb.enabled(true))
.source(SourceConfig.of(scf -> scf.filter(ff -> ff.includes(FieldNames.PATH).includes(new ArrayList<>(facetFields)))))
- .query(Query.of(qb -> qb.bool(builder.build())))
+ .query(Query.of(qb -> qb.bool(elasticRequestHandler.baseQueryBuilder().build())))
.aggregations(elasticRequestHandler.aggregations())
.size(sampleSize)
+ .sort(s ->
+ s.field(fs -> fs.field(
+ ElasticIndexDefinition.PATH_RANDOM_VALUE)
+ // this will handle the case when the field is not present in the index
+ .unmappedType(FieldType.Integer)
+ )
+ )
);
LOG.trace("Kicking search query with random sampling {}", searchRequest);
diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
index 595a2b4579..9d46e28f24 100644
--- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
+++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/FulltextIndexConstants.java
@@ -443,6 +443,6 @@ public interface FulltextIndexConstants {
* needed from an outside process that does not have visibility to the specific index module.
*/
Map<String, String> INDEX_VERSION_BY_TYPE = Map.of(
- "elasticsearch", "1.1.0"
+ "elasticsearch", "1.2.0"
);
}