You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ca...@apache.org on 2019/01/29 11:28:06 UTC
svn commit: r1852434 - in /jackrabbit/oak/branches/1.10: ./
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/
oak-lucene/src/test/java/org/apa...
Author: catholicon
Date: Tue Jan 29 11:28:06 2019
New Revision: 1852434
URL: http://svn.apache.org/viewvc?rev=1852434&view=rev
Log:
OAK-8001: Lucene index can be empty (no :data node) in composite node store setup (backport r1852084 from trunk)
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactory.java
jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java
jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java
jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java
Propchange: jackrabbit/oak/branches/1.10/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Jan 29 11:28:06 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1851451,1852052
+/jackrabbit/oak/trunk:1851451,1852052,1852084
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactory.java?rev=1852434&r1=1852433&r2=1852434&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactory.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/reader/DefaultIndexReaderFactory.java Tue Jan 29 11:28:06 2019
@@ -72,7 +72,8 @@ public class DefaultIndexReaderFactory i
}
for (Mount m : mountInfoProvider.getNonDefaultMounts()) {
reader = createReader(m, definition, defnState, indexPath);
- if (reader != null) {
+ // Although, it shouldn't matter much, but let's not add extra readers if they don't have any docs
+ if (reader != null && reader.getReader() != null && reader.getReader().numDocs() > 0) {
readers.add(reader);
}
}
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java?rev=1852434&r1=1852433&r2=1852434&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriter.java Tue Jan 29 11:28:06 2019
@@ -30,6 +30,10 @@ import org.apache.jackrabbit.oak.spi.mou
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.lucene.index.IndexableField;
+import static com.google.common.collect.Iterables.concat;
+import static java.util.Collections.singleton;
+import static java.util.stream.StreamSupport.stream;
+
class MultiplexingIndexWriter implements LuceneIndexWriter {
private final MountInfoProvider mountInfoProvider;
private final DirectoryFactory directoryFactory;
@@ -73,6 +77,15 @@ class MultiplexingIndexWriter implements
@Override
public boolean close(long timestamp) throws IOException {
+ // explicitly get writers for mounts which haven't got writers even at close.
+ // This essentially ensures we respect DefaultIndexWriters#close's intent to
+ // create empty index even if nothing has been written during re-index.
+ stream(concat(singleton(mountInfoProvider.getDefaultMount()), mountInfoProvider.getNonDefaultMounts())
+ .spliterator(), false)
+ .filter(m -> reindex && !m.isReadOnly()) // only needed when re-indexing for read-write mounts.
+ // reindex for ro-mount doesn't make sense in this case anyway.
+ .forEach(m -> getWriter(m)); // open default writers for mounts that passed all our tests
+
boolean indexUpdated = false;
for (LuceneIndexWriter w : writers.values()) {
indexUpdated |= w.close(timestamp);
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java?rev=1852434&r1=1852433&r2=1852434&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/MultiplexingLucenePropertyIndexTest.java Tue Jan 29 11:28:06 2019
@@ -57,6 +57,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.DefaultIndexWriterFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriterConfig;
@@ -136,6 +137,34 @@ public class MultiplexingLucenePropertyI
.createContentRepository();
}
+ // OAK-8001
+ @Test
+ public void emptyIndex() throws Exception {
+ NodeBuilder defnBuilder = builder.child("oak:index").child("foo");
+ IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder(defnBuilder).noAsync();
+ idxBuilder.indexRule("nt:base").property("propa").propertyIndex();
+ idxBuilder.build();
+
+ LuceneIndexDefinition defn = new LuceneIndexDefinition(builder.getNodeState(), defnBuilder.getNodeState(),
+ "/oak:index/foo");
+
+ //1. reindex to initialize empty data directories under each mount
+ DefaultIndexWriterFactory factory = new DefaultIndexWriterFactory(mip,
+ new DefaultDirectoryFactory(null, null),
+ new LuceneIndexWriterConfig());
+ LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
+ writer.close(0);
+
+ //2. Construct the readers
+ LuceneIndexReaderFactory readerFactory = new DefaultIndexReaderFactory(mip, null);
+ List<LuceneIndexReader> readers = readerFactory.createReaders(defn, builder.getNodeState(), defn.getIndexPath());
+
+ LuceneIndexNodeManager nodeManager = new LuceneIndexNodeManager("foo", defn, readers, null);
+ LuceneIndexNode node = nodeManager.acquire();
+ assertEquals(0, node.getIndexStatistics().numDocs());
+ node.release();
+ }
+
@Test
public void numDocsIsSumOfAllReaders() throws Exception{
NodeBuilder defnBuilder = newLucenePropertyIndexDefinition(builder, "test", ImmutableSet.of("foo"), "async");
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java?rev=1852434&r1=1852433&r2=1852434&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/writer/MultiplexingIndexWriterTest.java Tue Jan 29 11:28:06 2019
@@ -50,11 +50,10 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
import static org.apache.jackrabbit.oak.InitialContentHelper.INITIAL_CONTENT;
import static org.hamcrest.CoreMatchers.instanceOf;
-import static org.hamcrest.Matchers.contains;
-import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
public class MultiplexingIndexWriterTest {
@Rule
@@ -64,9 +63,12 @@ public class MultiplexingIndexWriterTest
private NodeBuilder builder = EMPTY_NODE.builder();
private LuceneIndexDefinition defn = new LuceneIndexDefinition(root, builder.getNodeState(), "/foo");
private MountInfoProvider mip = Mounts.newBuilder()
- .mount("foo", "/libs", "/apps").build();
+ .mount("foo", "/libs", "/apps")
+ .readOnlyMount("ro", "/ro-tree")
+ .build();
private Mount fooMount;
+ private Mount roMount;
private Mount defaultMount;
private LuceneIndexWriterConfig writerConfig = new LuceneIndexWriterConfig();
@@ -86,6 +88,17 @@ public class MultiplexingIndexWriterTest
public void closeWithoutChange() throws Exception{
LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
+ assertTrue(writer.close(0));
+ assertEquals(2, Iterables.size(getIndexDirNodes()));
+ assertFalse(builder.hasChildNode(indexDirName(roMount)));
+ assertEquals(0, numDocs(fooMount));
+ assertEquals(0, numDocs(defaultMount));
+
+ // delete all dir nodes first
+ getIndexDirNodes().stream().forEach(dirName -> builder.getChildNode(dirName).remove());
+
+ // empty index dir doesn't get created during normal indexing
+ writer = factory.newInstance(defn, builder, false);
assertFalse(writer.close(0));
assertEquals(0, Iterables.size(getIndexDirNodes()));
}
@@ -95,21 +108,43 @@ public class MultiplexingIndexWriterTest
LuceneIndexWriterFactory factory = newDirectoryFactory();
LuceneIndexWriter writer = factory.newInstance(defn, builder, true);
+
//1. Add entry in foo mount
writer.updateDocument("/libs/config", newDoc("/libs/config"));
writer.close(0);
- List<String> names = getIndexDirNodes();
- //Only dirNode for mount foo should be present
- assertThat(names, contains(indexDirName(fooMount)));
+
+ // only mount foo must have indexed document
+ assertEquals(1, numDocs(fooMount));
+ assertEquals(0, numDocs(defaultMount));
//2. Add entry in default mount
writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content", newDoc("/content"));
writer.close(0);
- names = getIndexDirNodes();
- //Dir names for both mounts should be present
- assertThat(names, containsInAnyOrder(indexDirName(fooMount), indexDirName(defaultMount)));
+
+ // both mounts must have 1 document each
+ assertEquals(1, numDocs(fooMount));
+ assertEquals(1, numDocs(defaultMount));
+
+ //3. Add another entry in foo mount without reindexing
+ writer = factory.newInstance(defn, builder, false);
+ writer.updateDocument("/libs/config1", newDoc("/libs/config1"));
+ writer.close(0);
+
+ // only mount foo must have indexed document
+ assertEquals(2, numDocs(fooMount));
+ assertEquals(1, numDocs(defaultMount));
+
+ //4. Add another entry in default mount without reindexing
+ writer = factory.newInstance(defn, builder, false);
+ writer.updateDocument("/content1", newDoc("/content1"));
+ writer.close(0);
+
+
+ // both mounts must have 1 document each
+ assertEquals(2, numDocs(fooMount));
+ assertEquals(2, numDocs(defaultMount));
}
@Test
@@ -125,18 +160,39 @@ public class MultiplexingIndexWriterTest
//1. Add entry in foo mount
writer.updateDocument("/libs/config", newDoc("/libs/config"));
writer.close(0);
- List<String> names = getIndexDirNodes();
- //Only dirNode for mount foo should be present
- assertThat(names, contains(indexDirName(fooMount)));
+
+ // only mount foo must have indexed document
+ assertEquals(1, numDocs(fooMount));
+ assertEquals(0, numDocs(defaultMount));
//2. Add entry in default mount
writer = factory.newInstance(defn, builder, true);
writer.updateDocument("/content", newDoc("/content"));
writer.close(0);
- names = getIndexDirNodes();
- //Dir names for both mounts should be present
- assertThat(names, containsInAnyOrder(indexDirName(fooMount), indexDirName(defaultMount)));
+
+ // both mounts must have 1 document each
+ assertEquals(1, numDocs(fooMount));
+ assertEquals(1, numDocs(defaultMount));
+
+ //3. Add another entry in foo mount without reindexing
+ writer = factory.newInstance(defn, builder, false);
+ writer.updateDocument("/libs/config1", newDoc("/libs/config1"));
+ writer.close(0);
+
+ // only mount foo must have indexed document
+ assertEquals(2, numDocs(fooMount));
+ assertEquals(1, numDocs(defaultMount));
+
+ //4. Add another entry in default mount without reindexing
+ writer = factory.newInstance(defn, builder, false);
+ writer.updateDocument("/content1", newDoc("/content1"));
+ writer.close(0);
+
+
+ // both mounts must have 1 document each
+ assertEquals(2, numDocs(fooMount));
+ assertEquals(2, numDocs(defaultMount));
}
@Test
@@ -195,6 +251,7 @@ public class MultiplexingIndexWriterTest
private void initializeMounts() {
fooMount = mip.getMountByName("foo");
+ roMount = mip.getMountByName("ro");
defaultMount = mip.getDefaultMount();
}
@@ -215,6 +272,14 @@ public class MultiplexingIndexWriterTest
return names;
}
+ private void deleteIndexDirNodes(){
+ for (String name : builder.getChildNodeNames()) {
+ if (MultiplexersLucene.isIndexDirName(name)){
+ builder.getChildNode(name).remove();
+ }
+ }
+ }
+
private String indexDirName(Mount m){
return MultiplexersLucene.getIndexDirName(m);
}