You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2026/03/23 15:35:50 UTC
(jackrabbit-oak) branch OAK-12146 created (now a8af635025)
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a change to branch OAK-12146
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at a8af635025 OAK-12146 Use the latest elastic OR lucene index, if there are multiple versions
This branch includes the following new commits:
new a8af635025 OAK-12146 Use the latest elastic OR lucene index, if there are multiple versions
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 01/01: OAK-12146 Use the latest elastic OR lucene index, if there are multiple versions
Posted by th...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-12146
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit a8af6350251a4e5348e8bc590e546a8c0a16bdd1
Author: Thomas Mueller <mu...@adobe.com>
AuthorDate: Mon Mar 23 16:35:34 2026 +0100
OAK-12146 Use the latest elastic OR lucene index, if there are multiple versions
---
.../jackrabbit/oak/plugins/index/IndexName.java | 31 +++
oak-it-osgi/pom.xml | 13 ++
.../oak/osgi/IndexVersionSelectionTest.java | 222 +++++++++++++++++++++
.../index/search/spi/query/FulltextIndex.java | 9 +
4 files changed, 275 insertions(+)
diff --git a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java
index 994fe62a79..b453804ebf 100644
--- a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java
+++ b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/IndexName.java
@@ -23,6 +23,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
@@ -234,6 +235,36 @@ public class IndexName implements Comparable<IndexName> {
return result;
}
+ /**
+ * Remove indexes if there is a newer, active version.
+ * This suppresses indexes when a newer version of a different type exists
+ * (e.g. lucene vs. elasticsearch).
+ *
+ * @param candidatePaths paths of one specific index type being evaluated
+ * @param allCompetingPaths paths of all competing index types (e.g. both lucene and elasticsearch)
+ * @return candidates that are not superseded by a higher-versioned entry in allCompetingPaths
+ */
+ public static Collection<String> filterGloballySuperseded(
+ Collection<String> candidatePaths, Collection<String> allCompetingPaths) {
+ Map<String, IndexName> maxByBase = new HashMap<>();
+ for (String p : allCompetingPaths) {
+ IndexName n = IndexName.parse(PathUtils.getName(p));
+ IndexName stored = maxByBase.get(n.baseName);
+ if (stored == null || stored.compareTo(n) < 0) {
+ maxByBase.put(n.baseName, n);
+ }
+ }
+ List<String> result = new ArrayList<>();
+ for (String p : candidatePaths) {
+ IndexName n = IndexName.parse(PathUtils.getName(p));
+ IndexName globalMax = maxByBase.get(n.baseName);
+ if (globalMax == null || globalMax.compareTo(n) == 0) {
+ result.add(p);
+ }
+ }
+ return result;
+ }
+
public static Collection<String> filterNewestIndexes(Collection<String> indexPaths) {
HashMap<String, IndexName> latestVersions = new HashMap<>();
for (String p : indexPaths) {
diff --git a/oak-it-osgi/pom.xml b/oak-it-osgi/pom.xml
index 4161e48ac8..be501a47f8 100644
--- a/oak-it-osgi/pom.xml
+++ b/oak-it-osgi/pom.xml
@@ -183,6 +183,19 @@
<version>${project.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.jackrabbit</groupId>
+ <artifactId>oak-search-elastic</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testcontainers</groupId>
+ <artifactId>elasticsearch</artifactId>
+ <version>${testcontainers.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.jackrabbit</groupId>
<artifactId>oak-store-composite</artifactId>
diff --git a/oak-it-osgi/src/test/java/org/apache/jackrabbit/oak/osgi/IndexVersionSelectionTest.java b/oak-it-osgi/src/test/java/org/apache/jackrabbit/oak/osgi/IndexVersionSelectionTest.java
new file mode 100644
index 0000000000..180b7f5eec
--- /dev/null
+++ b/oak-it-osgi/src/test/java/org/apache/jackrabbit/oak/osgi/IndexVersionSelectionTest.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.osgi;
+
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.ContentSession;
+import org.apache.jackrabbit.oak.api.QueryEngine;
+import org.apache.jackrabbit.oak.api.Result;
+import org.apache.jackrabbit.oak.api.Root;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnection;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticConnectionRule;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexNameHelper;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticIndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.elastic.ElasticMetricHandler;
+import org.apache.jackrabbit.oak.plugins.index.elastic.query.ElasticIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.elastic.util.ElasticIndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.LuceneIndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.search.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NAME;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests that verify index version selection behavior when multiple versions of an index exist
+ * across different index types (lucene and elasticsearch).
+ */
+public class IndexVersionSelectionTest {
+
+ @ClassRule
+ public static ElasticConnectionRule elasticRule =
+ new ElasticConnectionRule(System.getProperty("elasticConnectionString"));
+
+ private ElasticConnection elasticConnection;
+ private ContentRepository repo;
+ private ContentSession session;
+
+ @Before
+ public void setUp() throws Exception {
+ LuceneIndexProvider luceneProvider = new LuceneIndexProvider();
+ LuceneIndexEditorProvider luceneEditorProvider = new LuceneIndexEditorProvider();
+
+ elasticConnection = elasticRule.useDocker()
+ ? elasticRule.getElasticConnectionForDocker()
+ : elasticRule.getElasticConnectionFromString();
+ ElasticIndexTracker elasticTracker = new ElasticIndexTracker(
+ elasticConnection,
+ new ElasticMetricHandler(StatisticsProvider.NOOP));
+ ElasticIndexProvider elasticProvider = new ElasticIndexProvider(elasticTracker);
+ // ElasticIndexEditorProvider is intentionally not registered here. See the comment
+ // in testLatestVersionIsUsedEvenWithHigherCost() for the full explanation.
+
+ repo = new Oak(new MemoryNodeStore())
+ .with(new InitialContent())
+ .with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) luceneProvider)
+ .with((Observer) luceneProvider)
+ .with(luceneEditorProvider)
+ .with((QueryIndexProvider) elasticProvider)
+ .with((Observer) elasticTracker)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .createContentRepository();
+
+ session = repo.login(null, null);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (session != null) {
+ session.close();
+ }
+ if (elasticConnection != null) {
+ elasticConnection.close();
+ }
+ }
+
+ /**
+ * Returns a fresh index definition builder for the given type, configured with
+ * {@code noAsync()} so that the index definition is registered in the respective
+ * index tracker via the Observer callback during commit, making query plans
+ * available immediately after {@code root.commit()} returns.
+ */
+ private IndexDefinitionBuilder newBuilder(String type) {
+ if ("lucene".equals(type)) {
+ return new LuceneIndexDefinitionBuilder().noAsync();
+ }
+ return new ElasticIndexDefinitionBuilder().noAsync();
+ }
+
+ /**
+ * Verifies that the latest index version (asset-10-custom-3, of type {@code newType}) is
+ * selected even when its cost is set to a very high value (1 million), proving that
+ * version-based selection takes precedence over cost. The index family is:
+ * <pre>
+ * asset-10 (oldType)
+ * asset-10-custom-1 (oldType)
+ * asset-10-custom-2 (oldType)
+ * asset-10-custom-3 (newType, costPerEntry=costPerExecution=1_000_000)
+ * </pre>
+ * All indexes carry the tag "myTag" and use selectionPolicy="tag". The query uses
+ * {@code option(index tag myTag)} and is a {@code contains()} query so that traversal
+ * is not an option. Version selection must filter all but asset-10-custom-3 from the
+ * candidate list regardless of its higher cost.
+ */
+ private void testLatestVersionIsUsedEvenWithHigherCost(String oldType, String newType)
+ throws Exception {
+ Root root = session.getLatestRoot();
+ Tree oakIndex = root.getTree("/" + INDEX_DEFINITIONS_NAME);
+
+ for (String name : new String[]{"asset-10", "asset-10-custom-1", "asset-10-custom-2"}) {
+ IndexDefinitionBuilder b = newBuilder(oldType);
+ b.tags("myTag", oldType);
+ b.selectionPolicy("tag");
+ b.indexRule("nt:base").property("asset").analyzed();
+ b.build(oakIndex.addChild(name));
+ }
+
+ // asset-10-custom-3: same setup but with a very high cost. Version selection must
+ // still pick this index (it is the latest) and not fall back to a cheaper older version.
+ IndexDefinitionBuilder b = newBuilder(newType);
+ b.tags("myTag", newType);
+ b.selectionPolicy("tag");
+ b.indexRule("nt:base").property("asset").analyzed();
+ Tree custom3Tree = oakIndex.addChild("asset-10-custom-3");
+ b.build(custom3Tree);
+ custom3Tree.setProperty(FulltextIndexConstants.COST_PER_ENTRY, 1_000_000.0);
+ custom3Tree.setProperty(FulltextIndexConstants.COST_PER_EXECUTION, 1_000_000.0);
+
+ root.getTree("/").addChild("content").setProperty("asset", "test-value");
+ root.commit();
+
+ // Why we create the ES index manually instead of registering ElasticIndexEditorProvider:
+ //
+ // Normally, ElasticIndexEditorProvider would create the Elasticsearch index during
+ // root.commit(). However, this module's test classpath has a Lucene version conflict:
+ // oak-lucene depends on Lucene 4.7.2, while oak-search-elastic requires Lucene 9.x.
+ // Maven's dependency resolution picks lucene-core:4.7.2 (the nearer declaration), so
+ // lucene-core:9.x is absent. ElasticIndexEditorProvider transitively loads
+ // ElasticCustomAnalyzer, which imports org.apache.lucene.util.ResourceLoader — a class
+ // that only exists in lucene-core 9.x (in 4.x it had a different package). The result is
+ // a NoClassDefFoundError at commit time, even when no custom analyzers are configured.
+ //
+ // This test does not need to index actual content — it only checks which index the query
+ // planner *selects* (via EXPLAIN). For plan generation, ElasticIndexStatistics.numDocs()
+ // is called to estimate the entry count; it issues a COUNT request to Elasticsearch and
+ // throws index_not_found_exception if the index does not exist. That exception propagates
+ // as UncheckedExecutionException and is caught in FulltextIndex.getPlans(), which silently
+ // skips the index — causing the test to fail with "traverse allNodes".
+ //
+ // The fix: after committing the Oak index definitions (which registers the definition in
+ // the ElasticIndexTracker via the Observer callback), we create an empty Elasticsearch
+ // index directly via the REST client. numDocs() then returns 0, the planner can generate
+ // a plan, and version selection is exercised correctly.
+ String elasticName;
+ if ("elasticsearch".equals(newType)) {
+ elasticName = "asset-10-custom-3";
+ } else {
+ elasticName = "asset-10-custom-2";
+ }
+ String alias = ElasticIndexNameHelper.getElasticSafeIndexName(
+ elasticConnection.getIndexPrefix(),
+ "/" + INDEX_DEFINITIONS_NAME + "/" + elasticName);
+ elasticConnection.getClient().indices().create(c -> c.index(alias));
+
+ root = session.getLatestRoot();
+ Result result = root.getQueryEngine().executeQuery(
+ "explain select * from [nt:base] where contains([asset], 'test-value')" +
+ " option(index tag myTag)",
+ "JCR-SQL2",
+ QueryEngine.NO_BINDINGS,
+ QueryEngine.NO_MAPPINGS);
+
+ String plan = result.getRows().iterator().next().getValue("plan").getValue(Type.STRING);
+
+ // Version selection keeps only asset-10-custom-3 (the latest). The contains() constraint
+ // prevents traversal, so the high-cost index must be used.
+ assertTrue("Expected asset-10-custom-3 to be used, but got: " + plan,
+ plan.contains("asset-10-custom-3"));
+ }
+
+ @Test
+ public void latestLuceneVersionIsUsedEvenWithHigherCost() throws Exception {
+ testLatestVersionIsUsedEvenWithHigherCost("elasticsearch", "lucene");
+ }
+
+ @Test
+ public void latestElasticsearchVersionIsUsedEvenWithHigherCost() throws Exception {
+ testLatestVersionIsUsedEvenWithHigherCost("lucene", "elasticsearch");
+ }
+}
diff --git a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
index 340e5b2744..ec67f28999 100644
--- a/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
+++ b/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndex.java
@@ -56,8 +56,10 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
+import java.util.Set;
import java.util.function.Predicate;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.TYPE_PROPERTY_NAME;
import static org.apache.jackrabbit.oak.spi.query.QueryIndex.AdvancedQueryIndex;
import static org.apache.jackrabbit.oak.spi.query.QueryIndex.NativeQueryIndex;
@@ -76,6 +78,9 @@ public abstract class FulltextIndex implements AdvancedQueryIndex, QueryIndex, N
private static final double MIN_COST = 2.1;
+ // Index types that may compete; other types (e.g. "disabled") are excluded
+ private static final Set<String> COMPETING_INDEX_TYPES = Set.of("lucene", "elasticsearch");
+
protected abstract IndexNode acquireIndexNode(String indexPath);
protected abstract String getType();
@@ -121,6 +126,10 @@ public abstract class FulltextIndex implements AdvancedQueryIndex, QueryIndex, N
} else {
indexPaths = IndexName.filterNewestIndexes(indexPaths);
}
+ Collection<String> allCompetingPaths = new IndexLookup(rootState,
+ state -> COMPETING_INDEX_TYPES.contains(state.getString(TYPE_PROPERTY_NAME)))
+ .collectIndexNodePaths(filter);
+ indexPaths = IndexName.filterGloballySuperseded(indexPaths, allCompetingPaths);
List<IndexPlan> plans = new ArrayList<>(indexPaths.size());
for (String path : indexPaths) {
IndexNode indexNode = null;