You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/10/04 13:21:42 UTC
svn commit: r1811071 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/
Author: chetanm
Date: Wed Oct 4 13:21:42 2017
New Revision: 1811071
URL: http://svn.apache.org/viewvc?rev=1811071&view=rev
Log:
OAK-6777 - IndexReader closed exception in previous reader
- Add ignored testcase
- Also added another test in NRTIndexFactory test which ensures
that older NRTIndex reader get initialized and then close works
as expected
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java?rev=1811071&r1=1811070&r2=1811071&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DocumentQueue.java Wed Oct 4 13:21:42 2017
@@ -64,6 +64,7 @@ public class DocumentQueue implements Cl
private final MeterStats added;
private final MeterStats dropped;
private final Striped<Lock> locks = Striped.lock(64);
+ private UncaughtExceptionHandler delegate = (t, e) -> {};
/**
* Time in millis for which add call to queue
@@ -124,6 +125,7 @@ public class DocumentQueue implements Cl
PERF_LOGGER.end(start, 1, "Processed {} docs from queue", count);
} catch (Throwable t) {
exceptionHandler.uncaughtException(Thread.currentThread(), t);
+ delegate.uncaughtException(Thread.currentThread(), t);
}
return null;
}
@@ -198,6 +200,14 @@ public class DocumentQueue implements Cl
addDocsToIndex(docsPerIndex, false);
}
+ /**
+ * Delegate handled which can be used by test to check for
+ * any exception occurring in queue processing
+ */
+ public void setExceptionHandler(UncaughtExceptionHandler delegate) {
+ this.delegate = delegate;
+ }
+
private void addDocsToIndex(Map<String, Collection<LuceneDoc>> docsPerIndex, boolean docsFromQueue) {
//If required it can optimized by indexing diff indexes in parallel
//Something to consider if it becomes a bottleneck
@@ -271,6 +281,7 @@ public class DocumentQueue implements Cl
//For now we just log it. Later we need to see if frequent error then to
//temporarily disable indexing for this index
log.warn("Error occurred while indexing index [{}]",indexPath, e);
+ delegate.uncaughtException(Thread.currentThread(), e);
} finally {
indexNode.release();
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java?rev=1811071&r1=1811070&r2=1811071&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndexFactoryTest.java Wed Oct 4 13:21:42 2017
@@ -25,10 +25,12 @@ import java.io.IOException;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.IndexingMode;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorContext;
import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.lucene.document.Document;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
@@ -36,6 +38,7 @@ import org.junit.rules.TemporaryFolder;
import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
import static org.apache.jackrabbit.oak.InitialContent.INITIAL_CONTENT;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.FieldFactory.newPathField;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -120,6 +123,28 @@ public class NRTIndexFactoryTest {
}
@Test
+ public void indexCreationAndCloserWithUpdate() throws Exception{
+ IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
+
+ Document d = new Document();
+ d.add(newPathField("/a/b"));
+
+ NRTIndex idx1 = indexFactory.createIndex(idxDefn);
+ idx1.getWriter().updateDocument("/a/b", d);
+ assertEquals(1, idx1.getReaders().size());
+
+ NRTIndex idx2 = indexFactory.createIndex(idxDefn);
+ idx2.getWriter().updateDocument("/a/b", d);
+ idx1.getWriter().updateDocument("/a/b", d);
+ assertEquals(2, idx2.getReaders().size());
+
+ NRTIndex idx3 = indexFactory.createIndex(idxDefn);
+ NRTIndex idx4 = indexFactory.createIndex(idxDefn);
+ assertTrue(idx1.isClosed());
+ }
+
+
+ @Test
public void closeIndexOnClose() throws Exception{
IndexDefinition idxDefn = getNRTIndexDefinition("/foo");
@@ -139,7 +164,7 @@ public class NRTIndexFactoryTest {
private IndexDefinition getIndexDefinition(String indexPath, IndexingMode indexingMode) {
TestUtil.enableIndexingMode(builder, indexingMode);
-
+ LuceneIndexEditorContext.configureUniqueId(builder);
return new IndexDefinition(root, builder.getNodeState(), indexPath);
}
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java?rev=1811071&r1=1811070&r2=1811071&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/ReaderRefCountIT.java Wed Oct 4 13:21:42 2017
@@ -34,6 +34,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexNode;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorContext;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -43,6 +44,7 @@ import org.apache.jackrabbit.oak.stats.S
import org.apache.lucene.document.Document;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.junit.Before;
+import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -61,7 +63,7 @@ public class ReaderRefCountIT {
private NodeState root = INITIAL_CONTENT;
private IndexCopier indexCopier;
- private int runTimeInSecs = 50;
+ private int runTimeInSecs = 25;
private int noOfThread = 5;
@Before
@@ -76,7 +78,7 @@ public class ReaderRefCountIT {
idx.async("async", "sync");
NRTIndexFactory nrtFactory = new NRTIndexFactory(indexCopier, StatisticsProvider.NOOP);
- runMultiReaderScenario(idx, nrtFactory);
+ runMultiReaderScenario(idx, nrtFactory, false);
}
@Test
@@ -87,13 +89,31 @@ public class ReaderRefCountIT {
NRTIndexFactory nrtFactory = new NRTIndexFactory(indexCopier, Clock.SIMPLE,
0 , StatisticsProvider.NOOP);
- runMultiReaderScenario(idx, nrtFactory);
+ runMultiReaderScenario(idx, nrtFactory, false);
+ }
+
+ /**
+ * This test enables 1 more thread which updates the IndexTracker
+ * This causes the IndexNodeManager to switch to newer indexes
+ * and hence lead to creation and closing of older NRTIndexes
+ */
+ @Ignore("OAK-6777")
+ @Test
+ public void indexTrackerUpdatesAndNRT() throws Exception{
+ IndexDefinitionBuilder idx = new IndexDefinitionBuilder();
+ idx.indexRule("nt:base").property("foo").propertyIndex();
+ idx.async("async", "nrt");
+
+ NRTIndexFactory nrtFactory = new NRTIndexFactory(indexCopier, Clock.SIMPLE,
+ 0 , StatisticsProvider.NOOP);
+ runMultiReaderScenario(idx, nrtFactory, true);
}
private void runMultiReaderScenario(IndexDefinitionBuilder defnb,
- NRTIndexFactory nrtFactory) throws Exception{
+ NRTIndexFactory nrtFactory, boolean updateIndex) throws Exception{
NodeBuilder builder = root.builder();
builder.child("oak:index").setChildNode("fooIndex", defnb.build());
+ LuceneIndexEditorContext.configureUniqueId(builder.child("oak:index").child("fooIndex"));
NodeState repoState = builder.getNodeState();
String indexPath = "/oak:index/fooIndex";
@@ -104,9 +124,20 @@ public class ReaderRefCountIT {
IndexTracker tracker = new IndexTracker(new DefaultIndexReaderFactory(defaultMountInfoProvider(), indexCopier), nrtFactory);
tracker.update(repoState);
+ CountDownLatch errorLatch = new CountDownLatch(1);
+ UncaughtExceptionHandler uh = new UncaughtExceptionHandler() {
+ @Override
+ public void uncaughtException(Thread t, Throwable e) {
+ e.printStackTrace();
+ exceptionList.add(e);
+ errorLatch.countDown();
+ }
+ };
+
DocumentQueue queue = new DocumentQueue(100, tracker, sameThreadExecutor());
+ queue.setExceptionHandler(uh);
+
- CountDownLatch errorLatch = new CountDownLatch(1);
//Writer should try to refresh same IndexNode within same lock
//i.e. simulate a scenario where DocumentQueue pushes multiple
//sync index docs in same commit
@@ -141,12 +172,15 @@ public class ReaderRefCountIT {
}
};
- UncaughtExceptionHandler uh = new UncaughtExceptionHandler() {
+ Runnable indexUpdater = new Runnable() {
@Override
- public void uncaughtException(Thread t, Throwable e) {
- e.printStackTrace();
- exceptionList.add(e);
- errorLatch.countDown();
+ public void run() {
+ int count = 0;
+ while(!stop.get()) {
+ NodeBuilder b = repoState.builder();
+ b.getChildNode("oak:index").getChildNode("fooIndex").setProperty("count", count++);
+ tracker.update(b.getNodeState());
+ }
}
};
@@ -159,6 +193,10 @@ public class ReaderRefCountIT {
t.setUncaughtExceptionHandler(uh);
}
+ if (updateIndex) {
+ threads.add(new Thread(indexUpdater));
+ }
+
for (Thread t : threads) {
t.start();
}