You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2019/01/24 09:41:02 UTC
svn commit: r1852007 - in /jackrabbit/oak/trunk:
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
oak-lucene/src/test/java/org/apache/jackrabbit/...
Author: thomasm
Date: Thu Jan 24 09:41:02 2019
New Revision: 1852007
URL: http://svn.apache.org/viewvc?rev=1852007&view=rev
Log:
OAK-7947 Lazy loading of Lucene index files startup
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java
jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexTracker.java Thu Jan 24 09:41:02 2019
@@ -40,6 +40,7 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -195,39 +196,29 @@ public class IndexTracker {
refresh = true;
}
+ /**
+ * Acquire the index node, if the index is good.
+ *
+ * @param path the index path
+ * @return the index node, or null if it's a bad (corrupt) index
+ */
+ @Nullable
public LuceneIndexNode acquireIndexNode(String path) {
LuceneIndexNodeManager index = indices.get(path);
LuceneIndexNode indexNode = index != null ? index.acquire() : null;
if (indexNode != null) {
return indexNode;
- } else {
- return findIndexNode(path);
}
+ return findIndexNode(path);
}
+ /**
+ * Get the index node, if the index is good.
+ *
+ * @param path the index path
+ * @return the index node, or null if it's a bad (corrupt) index
+ */
@Nullable
- public LuceneIndexDefinition getIndexDefinition(String indexPath){
- LuceneIndexNodeManager node = indices.get(indexPath);
- if (node != null){
- //Accessing the definition should not require
- //locking as its immutable state
- return node.getDefinition();
- }
- return null;
- }
-
- public Set<String> getIndexNodePaths(){
- return indices.keySet();
- }
-
- public BadIndexTracker getBadIndexTracker() {
- return badIndexTracker;
- }
-
- public NodeState getRoot() {
- return root;
- }
-
private synchronized LuceneIndexNode findIndexNode(String path) {
// Retry the lookup from acquireIndexNode now that we're
// synchronized. The acquire() call is guaranteed to succeed
@@ -270,6 +261,42 @@ public class IndexTracker {
return null;
}
+ @Nullable
+ public LuceneIndexDefinition getIndexDefinition(String indexPath){
+ LuceneIndexNodeManager indexNodeManager = indices.get(indexPath);
+ if (indexNodeManager != null) {
+ // Accessing the definition should not require
+ // locking as its immutable state
+ return indexNodeManager.getDefinition();
+ }
+ // fallback - create definition from scratch
+ NodeState node = NodeStateUtils.getNode(root, indexPath);
+ if (!node.exists()) {
+ return null;
+ }
+ // only if there exists a stored index definition
+ if (!node.hasChildNode(INDEX_DEFINITION_NODE)) {
+ return null;
+ }
+ if (!isLuceneIndexNode(node)) {
+ return null;
+ }
+ // this will internally use the stored index definition
+ return new LuceneIndexDefinition(root, node, indexPath);
+ }
+
+ public Set<String> getIndexNodePaths(){
+ return indices.keySet();
+ }
+
+ public BadIndexTracker getBadIndexTracker() {
+ return badIndexTracker;
+ }
+
+ public NodeState getRoot() {
+ return root;
+ }
+
private static boolean isStatusChanged(NodeState before, NodeState after) {
return !EqualsDiff.equals(before.getChildNode(STATUS_NODE), after.getChildNode(STATUS_NODE));
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndex.java Thu Jan 24 09:41:02 2019
@@ -219,14 +219,17 @@ public class LuceneIndex implements Adva
try{
if (node != null){
IndexDefinition defn = node.getDefinition();
- return Collections.singletonList(planBuilder(filter)
- .setEstimatedEntryCount(defn.getFulltextEntryCount(node.getIndexStatistics().numDocs()))
- .setCostPerExecution(defn.getCostPerExecution())
- .setCostPerEntry(defn.getCostPerEntry())
- .setDeprecated(defn.isDeprecated())
- .setAttribute(ATTR_INDEX_PATH, indexPath)
- .setDeprecated(defn.isDeprecated())
- .build());
+ LuceneIndexStatistics stats = node.getIndexStatistics();
+ if (stats != null) {
+ return Collections.singletonList(planBuilder(filter)
+ .setEstimatedEntryCount(defn.getFulltextEntryCount(stats.numDocs()))
+ .setCostPerExecution(defn.getCostPerExecution())
+ .setCostPerEntry(defn.getCostPerEntry())
+ .setDeprecated(defn.isDeprecated())
+ .setAttribute(ATTR_INDEX_PATH, indexPath)
+ .setDeprecated(defn.isDeprecated())
+ .build());
+ }
}
//No index node then no plan possible
return Collections.emptyList();
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexEditorProvider.java Thu Jan 24 09:41:02 2019
@@ -65,6 +65,7 @@ import static org.apache.jackrabbit.oak.
*
*/
public class LuceneIndexEditorProvider implements IndexEditorProvider {
+
private final Logger log = LoggerFactory.getLogger(getClass());
private final IndexCopier indexCopier;
private final ExtractedTextCache extractedTextCache;
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexMBeanImpl.java Thu Jan 24 09:41:02 2019
@@ -93,6 +93,8 @@ import static org.apache.jackrabbit.oak.
public class LuceneIndexMBeanImpl extends AnnotatedStandardMBean implements LuceneIndexMBean {
+ private static final boolean LOAD_INDEX_FOR_STATS = Boolean.parseBoolean(System.getProperty("oak.lucene.LoadIndexForStats", "false"));
+
private final Logger log = LoggerFactory.getLogger(getClass());
private final IndexTracker indexTracker;
private final NodeStore nodeStore;
@@ -381,11 +383,21 @@ public class LuceneIndexMBeanImpl extend
@Override
public String getSize(String indexPath) throws IOException {
+ if (!LOAD_INDEX_FOR_STATS) {
+ if (!indexTracker.getIndexNodePaths().contains(indexPath)) {
+ return "-1";
+ }
+ }
return String.valueOf(getIndexStats(indexPath).indexSize);
}
@Override
public String getDocCount(String indexPath) throws IOException {
+ if (!LOAD_INDEX_FOR_STATS) {
+ if (!indexTracker.getIndexNodePaths().contains(indexPath)) {
+ return "-1";
+ }
+ }
return String.valueOf(getIndexStats(indexPath).numDocs);
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNode.java Thu Jan 24 09:41:02 2019
@@ -37,6 +37,7 @@ public interface LuceneIndexNode extends
IndexSearcher getSearcher();
+ @Nullable
LuceneIndexStatistics getIndexStatistics();
List<LuceneIndexReader> getPrimaryReaders();
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Thu Jan 24 09:41:02 2019
@@ -29,6 +29,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.Predicate;
@@ -46,11 +47,13 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.PerfLogger;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.fv.SimSearchUtils;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
import org.apache.jackrabbit.oak.plugins.index.search.FieldNames;
import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule;
import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexLookup;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.spi.FulltextQueryTermsProvider;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.FacetHelper;
@@ -120,6 +123,8 @@ import org.apache.lucene.search.highligh
import org.apache.lucene.search.postingshighlight.PostingsHighlighter;
import org.apache.lucene.search.spell.SuggestWord;
import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.search.suggest.analyzing.AnalyzingInfixSuggester;
+import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -186,6 +191,9 @@ import static org.apache.lucene.search.B
*/
public class LucenePropertyIndex extends FulltextIndex {
+
+ private static boolean NON_LAZY = Boolean.getBoolean("oak.lucene.nonLazyIndex");
+
private static double MIN_COST = 2.1;
private static final Logger LOG = LoggerFactory
@@ -689,7 +697,10 @@ public class LucenePropertyIndex extends
@Override
protected LuceneIndexNode acquireIndexNode(String indexPath) {
- return tracker.acquireIndexNode(indexPath);
+ if (NON_LAZY) {
+ return tracker.acquireIndexNode(indexPath);
+ }
+ return new LazyLuceneIndexNode(tracker, indexPath);
}
@Override
@@ -1552,4 +1563,128 @@ public class LucenePropertyIndex extends
return null;
}
}
+
+ /**
+ * A index node implementation that acquires the underlying index only if
+ * actually needed. This is to avoid downloading the index for the planning
+ * phase, if there is no chance that the index is actually used.
+ */
+ static class LazyLuceneIndexNode implements LuceneIndexNode {
+ private AtomicBoolean released = new AtomicBoolean();
+ private IndexTracker tracker;
+ private String indexPath;
+ private volatile LuceneIndexNode indexNode;
+
+ LazyLuceneIndexNode(IndexTracker tracker, String indexPath) {
+ this.tracker = tracker;
+ this.indexPath = indexPath;
+ }
+
+ @Override
+ public void release() {
+ if (released.getAndSet(true)) {
+ // already released
+ return;
+ }
+ if (indexNode != null) {
+ indexNode.release();
+ }
+ // to ensure it is not used after releasing
+ indexNode = null;
+ tracker = null;
+ indexPath = null;
+ }
+
+ private void checkNotReleased() {
+ if (released.get()) {
+ throw new IllegalStateException("Already released");
+ }
+ }
+
+ @Override
+ public LuceneIndexDefinition getDefinition() {
+ checkNotReleased();
+ return tracker.getIndexDefinition(indexPath);
+ }
+
+ private LuceneIndexNode getIndexNode() {
+ LuceneIndexNode n = findIndexNode();
+ if (n == null) {
+ String message = "No index node, corrupt index? " + indexPath;
+ LOG.warn(message);
+ throw new IllegalStateException(message);
+ }
+ return n;
+ }
+
+ @Nullable
+ private LuceneIndexNode findIndexNode() {
+ checkNotReleased();
+ LuceneIndexNode n = indexNode;
+ // double checked locking implemented in the correct way for Java 5
+ // and newer (actually I don't think this is ever called
+ // concurrently right now, but better be save)
+ if (n == null) {
+ synchronized (this) {
+ n = indexNode;
+ if (n == null) {
+ n = indexNode = tracker.acquireIndexNode(indexPath);
+ }
+ }
+ }
+ return n;
+ }
+
+ @Override
+ public int getIndexNodeId() {
+ return getIndexNode().getIndexNodeId();
+ }
+
+ @Nullable
+ @Override
+ public LuceneIndexStatistics getIndexStatistics() {
+ LuceneIndexNode n = findIndexNode();
+ if (n == null) {
+ return null;
+ }
+ return n.getIndexStatistics();
+ }
+
+ @Override
+ public IndexSearcher getSearcher() {
+ return getIndexNode().getSearcher();
+ }
+
+ @Override
+ public List<LuceneIndexReader> getPrimaryReaders() {
+ return getIndexNode().getPrimaryReaders();
+ }
+
+ @Override
+ public @Nullable Directory getSuggestDirectory() {
+ return getIndexNode().getSuggestDirectory();
+ }
+
+ @Override
+ public List<LuceneIndexReader> getNRTReaders() {
+ return getIndexNode().getNRTReaders();
+ }
+
+ @Override
+ public @Nullable AnalyzingInfixSuggester getLookup() {
+ return getIndexNode().getLookup();
+ }
+
+ @Override
+ public @Nullable LuceneIndexWriter getLocalWriter() throws IOException {
+ return getIndexNode().getLocalWriter();
+ }
+
+ @Override
+ public void refreshReadersOnWriteIfRequired() {
+ getIndexNode().refreshReadersOnWriteIfRequired();
+ }
+
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/OakStreamingIndexFile.java Thu Jan 24 09:41:02 2019
@@ -197,7 +197,7 @@ class OakStreamingIndexFile implements O
} else if (pos < position) {
LOG.warn("Seeking back on streaming index file {}. Current position {}, requested position {}. " +
"Please make sure that CopyOnRead and prefetch of index files are enabled.",
- getName(), position(), pos);
+ dirDetails + "/" + getName(), position(), pos);
// seeking back on input stream. Close current one
IOUtils.closeQuietly(blobInputStream);
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexTest.java Thu Jan 24 09:41:02 2019
@@ -24,10 +24,6 @@ import static com.google.common.collect.
import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
import static java.util.Arrays.asList;
import static javax.jcr.PropertyType.TYPENAME_STRING;
-import static junit.framework.Assert.assertEquals;
-import static junit.framework.Assert.assertFalse;
-import static junit.framework.Assert.assertTrue;
-import static junit.framework.Assert.fail;
import static org.apache.jackrabbit.JcrConstants.JCR_DATA;
import static org.apache.jackrabbit.JcrConstants.JCR_SYSTEM;
import static org.apache.jackrabbit.JcrConstants.NT_BASE;
@@ -54,6 +50,10 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PERSISTENCE_NAME;
import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PERSISTENCE_PATH;
import static org.apache.jackrabbit.oak.plugins.memory.PropertyStates.createProperty;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndexTest.java Thu Jan 24 09:41:02 2019
@@ -1233,7 +1233,6 @@ public class LucenePropertyIndexTest ext
assertQuery("select [jcr:path] from [nt:base] where propa is not null", asList("/test/a", "/test/b", "/test/c"));
}
-
@Test
public void rangeQueriesWithDate() throws Exception {
Tree idx = createIndex("test1", of("propa", "propb"));
@@ -2491,13 +2490,13 @@ public class LucenePropertyIndexTest ext
prop1.setProperty(FulltextIndexConstants.PROP_PROPERTY_INDEX, true);
root.commit();
- //force CoR
- executeQuery("SELECT * FROM [mix:title]", SQL2);
+ // force CoR
+ executeQuery("select * from [mix:title] where [jcr:title] = 'x'", SQL2);
assertNotNull(corDir);
String localPathBeforeReindex = corDir;
- //CoW with re-indexing
+ // CoW with re-indexing
idx.setProperty("reindex", true);
root.commit();
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/property/SynchronousPropertyIndexTest.java Thu Jan 24 09:41:02 2019
@@ -264,6 +264,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a").setProperty("foo", "bar");
root.commit();
@@ -277,8 +278,8 @@ public class SynchronousPropertyIndexTes
assertQuery("select * from [nt:base] where [foo] = 'bar'", asList("/a", "/b"));
- //Do multiple runs which lead to path being returned from both property and lucene
- //index. But the actual result should only contain unique paths
+ // Do multiple runs which lead to path being returned from both property and lucene
+ // index. But the actual result should only contain unique paths
runAsyncIndex();
runAsyncIndex();
@@ -326,6 +327,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
assertThat(explain("select * from [nt:base] where [jcr:content/foo] = 'bar'"),
containsString("sync:(foo[jcr:content/foo] bar)"));
@@ -333,7 +335,6 @@ public class SynchronousPropertyIndexTes
containsString("sync:(foo bar)"));
}
-
@Test
public void relativePropertyTransform() throws Exception{
defnb.async("async", "nrt");
@@ -341,6 +342,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a/jcr:content").setProperty("foo", "bar");
createPath("/b").setProperty("foo", "bar");
@@ -360,6 +362,7 @@ public class SynchronousPropertyIndexTes
indexPath = "/content/oak:index/fooIndex";
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a").setProperty("foo", "bar");
createPath("/content/a").setProperty("foo", "bar");
@@ -427,7 +430,7 @@ public class SynchronousPropertyIndexTes
@Test
public void nodeTypeIndexing() throws Exception{
- registerTestNodTypes();
+ registerTestNodeTypes();
defnb.async("async", "nrt");
defnb.nodeTypeIndex();
@@ -435,6 +438,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a", "oak:TestSuperType");
createPath("/b", "oak:TestTypeB");
@@ -448,7 +452,7 @@ public class SynchronousPropertyIndexTes
@Test
public void nodeType_mixins() throws Exception{
- registerTestNodTypes();
+ registerTestNodeTypes();
defnb.async("async", "nrt");
defnb.nodeTypeIndex();
@@ -456,6 +460,7 @@ public class SynchronousPropertyIndexTes
addIndex(indexPath, defnb);
root.commit();
+ runAsyncIndex();
createPath("/a", "oak:Unstructured", singletonList("oak:TestMixA"));
createPath("/b", "oak:TestTypeB");
@@ -466,10 +471,10 @@ public class SynchronousPropertyIndexTes
assertQuery("select * from [oak:TestMixA]", asList("/a", "/b"));
}
- private void registerTestNodTypes() throws IOException, CommitFailedException {
+ private void registerTestNodeTypes() throws IOException, CommitFailedException {
optionalEditorProvider.delegate = new TypeEditorProvider();
NodeTypeRegistry.register(root, IOUtils.toInputStream(testNodeTypes, "utf-8"), "test nodeType");
- //Flush the changes to nodetypes
+ // Flush the changes to nodetypes
root.commit();
}
Modified: jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java (original)
+++ jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/IndexNode.java Thu Jan 24 09:41:02 2019
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.plugins.index.search;
+import org.jetbrains.annotations.Nullable;
+
/**
* Represents an instance of an index.
*
@@ -34,6 +36,7 @@ public interface IndexNode {
int getIndexNodeId();
+ @Nullable
IndexStatistics getIndexStatistics();
}
Modified: jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java?rev=1852007&r1=1852006&r2=1852007&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java (original)
+++ jackrabbit/oak/trunk/oak-search/src/main/java/org/apache/jackrabbit/oak/plugins/index/search/spi/query/FulltextIndexPlanner.java Thu Jan 24 09:41:02 2019
@@ -105,6 +105,11 @@ public class FulltextIndexPlanner {
}
public IndexPlan getPlan() {
+ if (definition == null) {
+ log.debug("Index {} not loaded", indexPath);
+ return null;
+ }
+
IndexPlan.Builder builder = getPlanBuilder();
if (definition.isTestMode()){
@@ -281,6 +286,9 @@ public class FulltextIndexPlanner {
costPerEntryFactor += sortOrder.size();
IndexPlan.Builder plan = defaultPlan();
+ if (plan == null) {
+ return null;
+ }
if (!sortOrder.isEmpty()) {
plan.setSortOrder(sortOrder);
}
@@ -413,9 +421,16 @@ public class FulltextIndexPlanner {
result.disableUniquePaths();
}
- //If native function can be handled by this index then ensure
- // that lowest cost if returned
- return canHandleNativeFunction ? defaultPlan().setEstimatedEntryCount(1) : null;
+ if (!canHandleNativeFunction) {
+ return null;
+ }
+ IndexPlan.Builder b = defaultPlan();
+ if (b == null) {
+ return null;
+ }
+ // If native function can be handled by this index, then ensure
+ // that lowest cost is returned
+ return b.setEstimatedEntryCount(1);
}
/*
@@ -713,7 +728,16 @@ public class FulltextIndexPlanner {
return rule.indexesAllNodesOfMatchingType() && !rule.isBasedOnNtBase();
}
+ @Nullable
private IndexPlan.Builder defaultPlan() {
+ // With OAK-7947 lucene indexes return a non-null index node to delay reading index files
+ // While IndexNode could have a status check method but for now we are using this work-around
+ // to check null on {@code getIndexStatistics()} as proxy indicator
+ // (this could be avoided by returning lazy statistics)
+ if (indexNode.getIndexStatistics() == null) {
+ return null;
+ }
+
return new IndexPlan.Builder()
.setCostPerExecution(definition.getCostPerExecution())
.setCostPerEntry(definition.getCostPerEntry())
@@ -730,11 +754,13 @@ public class FulltextIndexPlanner {
}
private long estimatedEntryCount() {
- int numOfDocs = getNumDocs();
if (useActualEntryCount) {
- return definition.isEntryCountDefined() ? definition.getEntryCount() : numOfDocs;
+ if (definition.isEntryCountDefined()) {
+ return definition.getEntryCount();
+ }
+ return getNumDocs();
} else {
- return estimatedEntryCount_Compat(numOfDocs);
+ return estimatedEntryCount_Compat(getNumDocs());
}
}
@@ -756,11 +782,20 @@ public class FulltextIndexPlanner {
}
private int getNumDocs() {
- return indexNode.getIndexStatistics().numDocs();
+ IndexStatistics indexStatistics = indexNode.getIndexStatistics();
+ if (indexStatistics == null) {
+ log.warn("Statistics not available - possibly index is corrupt? Returning high doc count");
+ return Integer.MAX_VALUE;
+ }
+ return indexStatistics.numDocs();
}
private int getMaxPossibleNumDocs(Map<String, PropertyDefinition> propDefns, Filter filter) {
IndexStatistics indexStatistics = indexNode.getIndexStatistics();
+ if (indexStatistics == null) {
+ log.warn("Statistics not available - possibly index is corrupt? Returning high doc count");
+ return Integer.MAX_VALUE;
+ }
int minNumDocs = indexStatistics.numDocs();
for (Map.Entry<String, PropertyDefinition> propDef : propDefns.entrySet()) {
String key = propDef.getKey();