You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mk...@apache.org on 2020/03/17 08:34:05 UTC
svn commit: r1875283 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/
Author: mkataria
Date: Tue Mar 17 08:34:04 2020
New Revision: 1875283
URL: http://svn.apache.org/viewvc?rev=1875283&view=rev
Log:
OAK-8898 On querying, IndexReader failed with AlreadyClosedException
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java?rev=1875283&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java Tue Mar 17 08:34:04 2020
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.index.lucene;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FacetTestHelper {
+ private static final Logger LOG = LoggerFactory.getLogger(FacetTestHelper.class);
+
+ public static void sleep(int timeInMillis) {
+ try {
+ if (timeInMillis > 0) {
+ LOG.info("Sleep time set to:" + timeInMillis + " ms");
+ Thread.sleep(timeInMillis);
+ }
+ } catch (InterruptedException e) {
+ LOG.error("Exception while thread sleep", e);
+ throw new RuntimeException(e);
+ }
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/FacetTestHelper.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java?rev=1875283&r1=1875282&r2=1875283&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexNodeManager.java Tue Mar 17 08:34:04 2020
@@ -64,6 +64,8 @@ public class LuceneIndexNodeManager {
private static final PerfLogger PERF_LOGGER =
new PerfLogger(LoggerFactory.getLogger(LuceneIndexNodeManager.class.getName() + ".perf"));
+ public final static String OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME = "oak.lucene.oldFacetProviderTestFailSleepInstrument";
+ private final static int OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_VALUE = Integer.getInteger(OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, 0);
static LuceneIndexNodeManager open(String indexPath, NodeState root, NodeState defnNodeState,
LuceneIndexReaderFactory readerFactory, @Nullable NRTIndexFactory nrtFactory)
@@ -113,10 +115,12 @@ public class LuceneIndexNodeManager {
private final Runnable refreshCallback = new Runnable() {
@Override
public void run() {
+ //Index reader gets closed when searching facets. This gets triggered in a multi threaded environment. Refer OAK-8898
+ FacetTestHelper.sleep(OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_VALUE);
if (refreshLock.tryAcquire()) {
try {
refreshReaders();
- }finally {
+ } finally {
refreshLock.release();
}
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java?rev=1875283&r1=1875282&r2=1875283&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LucenePropertyIndex.java Tue Mar 17 08:34:04 2020
@@ -52,6 +52,7 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.IndexingRule;
+import org.apache.jackrabbit.oak.plugins.index.search.IndexDefinition.SecureFacetConfiguration;
import org.apache.jackrabbit.oak.plugins.index.lucene.property.HybridPropertyIndexLookup;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReader;
import org.apache.jackrabbit.oak.plugins.index.lucene.score.ScorerProviderFactory;
@@ -193,6 +194,8 @@ public class LucenePropertyIndex extends
private static boolean NON_LAZY = Boolean.getBoolean("oak.lucene.nonLazyIndex");
+ public final static String OLD_FACET_PROVIDER_CONFIG_NAME = "oak.lucene.oldFacetProvider";
+ private final static boolean OLD_FACET_PROVIDER = Boolean.getBoolean(OLD_FACET_PROVIDER_CONFIG_NAME);
private static double MIN_COST = 2.1;
@@ -259,7 +262,7 @@ public class LucenePropertyIndex extends
private boolean noDocs = false;
private IndexSearcher indexSearcher;
private int indexNodeId = -1;
- private LuceneFacetProvider facetProvider = null;
+ private FacetProvider facetProvider;
private int rewoundCount = 0;
@Override
@@ -277,7 +280,7 @@ public class LucenePropertyIndex extends
}
private FulltextResultRow convertToRow(ScoreDoc doc, IndexSearcher searcher, Map<String, String> excerpts,
- LuceneFacetProvider facetProvider,
+ FacetProvider facetProvider,
String explanation) throws IOException {
IndexReader reader = searcher.getIndexReader();
//TODO Look into usage of field cache for retrieving the path
@@ -362,11 +365,18 @@ public class LucenePropertyIndex extends
PERF_LOGGER.end(start, -1, "{} ...", docs.scoreDocs.length);
nextBatchSize = (int) Math.min(nextBatchSize * 2L, 100000);
- long f = PERF_LOGGER.start();
if (facetProvider == null) {
- facetProvider = new LuceneFacetProvider(
- FacetHelper.getFacets(searcher, query, plan, indexNode.getDefinition().getSecureFacetConfiguration())
- );
+ long f = PERF_LOGGER.start();
+ if (OLD_FACET_PROVIDER) {
+ // here the current searcher gets referenced for later
+ // but the searcher might get closed in the meantime
+ facetProvider = new LuceneFacetProvider(
+ FacetHelper.getFacets(searcher, query, plan, indexNode.getDefinition().getSecureFacetConfiguration())
+ );
+ } else {
+ // a new searcher is opened and closed when needed
+ facetProvider = new DelayedLuceneFacetProvider(LucenePropertyIndex.this, query, plan, indexNode.getDefinition().getSecureFacetConfiguration());
+ }
PERF_LOGGER.end(f, -1, "facets retrieved");
}
@@ -1564,6 +1574,45 @@ public class LucenePropertyIndex extends
return Iterators.concat(propIndex.iterator(), itr);
}
+ static class DelayedLuceneFacetProvider implements FacetProvider {
+ private final LucenePropertyIndex index;
+ private final Query query;
+ private final IndexPlan plan;
+ private final SecureFacetConfiguration config;
+
+ DelayedLuceneFacetProvider(LucenePropertyIndex index, Query query, IndexPlan plan, SecureFacetConfiguration config) {
+ this.index = index;
+ this.query = query;
+ this.plan = plan;
+ this.config = config;
+ }
+
+ @Override
+ public List<Facet> getFacets(int numberOfFacets, String columnName) throws IOException {
+ LuceneIndexNode indexNode = index.acquireIndexNode(plan);
+ try {
+ IndexSearcher searcher = indexNode.getSearcher();
+ String facetFieldName = FulltextIndex.parseFacetField(columnName);
+ Facets facets = FacetHelper.getFacets(searcher, query, plan, config);
+ if (facets != null) {
+ ImmutableList.Builder<Facet> res = new ImmutableList.Builder<>();
+ FacetResult topChildren = facets.getTopChildren(numberOfFacets, facetFieldName);
+ if (topChildren != null) {
+ for (LabelAndValue lav : topChildren.labelValues) {
+ res.add(new Facet(
+ lav.label, lav.value.intValue()
+ ));
+ }
+ return res.build();
+ }
+ }
+ return null;
+ } finally {
+ indexNode.release();
+ }
+ }
+ }
+
static class LuceneFacetProvider implements FacetProvider {
private final Facets facets;
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java?rev=1875283&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java Tue Mar 17 08:34:04 2020
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexNodeManager;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.OptionalEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.Nullable;
+import org.junit.After;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import javax.jcr.GuestCredentials;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.RowIterator;
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+import static org.apache.jackrabbit.oak.spi.mount.Mounts.defaultMountInfoProvider;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class DelayedFacetReadTest extends AbstractQueryTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+ private static final int NUM_LABELS = 4;
+ private static final int NUM_LEAF_NODES = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ private static final String FACET_PROP = "facets";
+ private static final long REFRESH_DELTA = TimeUnit.SECONDS.toMillis(1);
+
+ private ExecutorService executorService = Executors.newFixedThreadPool(2);
+ private OptionalEditorProvider optionalEditorProvider = new OptionalEditorProvider();
+ private NRTIndexFactory nrtIndexFactory;
+ private LuceneIndexProvider luceneIndexProvider;
+ private NodeStore nodeStore;
+ private DocumentQueue queue;
+ private Clock clock = new Clock.Virtual();
+ private Whiteboard wb;
+ private QueryManager qm;
+ private Repository jcrRepo;
+ private Jcr jcr;
+ private Oak oak;
+ private Thread thread;
+ // backup original system properties i.e. before test started
+ private final Properties backupProperties = (Properties) System.getProperties().clone();
+
+ @After
+ public void tearDown() throws IOException {
+ luceneIndexProvider.close();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ new ExecutorCloser(executorService).close();
+ nrtIndexFactory.close();
+ // restore original system properties i.e. before test started
+ System.setProperties(backupProperties);
+ }
+
+ @Override
+ protected ContentRepository createRepository() {
+ IndexCopier copier;
+ try {
+ copier = new IndexCopier(executorService, temporaryFolder.getRoot());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ MountInfoProvider mip = defaultMountInfoProvider();
+ nrtIndexFactory = new NRTIndexFactory(copier, clock, TimeUnit.MILLISECONDS.toSeconds(REFRESH_DELTA), StatisticsProvider.NOOP);
+ nrtIndexFactory.setAssertAllResourcesClosed(true);
+ LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
+ IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
+ luceneIndexProvider = new LuceneIndexProvider(tracker);
+ queue = new DocumentQueue(100, tracker, sameThreadExecutor());
+ LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier,
+ tracker,
+ null,
+ null,
+ mip);
+ editorProvider.setIndexingQueue(queue);
+ LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, StatisticsProvider.NOOP);
+ nodeStore = new MemoryNodeStore();
+ oak = new Oak(nodeStore)
+ .with(new InitialContent())
+ .with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) luceneIndexProvider)
+ .with((Observer) luceneIndexProvider)
+ .with(localIndexObserver)
+ .with(editorProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(optionalEditorProvider)
+ .with(new NodeCounterEditorProvider())
+ //Effectively disable async indexing auto run
+ //such that we can control run timing as per test requirement
+ .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1));
+
+ wb = oak.getWhiteboard();
+ ContentRepository repo = oak.createContentRepository();
+ return repo;
+ }
+
+ private void createSmallDataset(int k) throws RepositoryException {
+ Random rGen = new Random(42);
+ Tree par = createPath("/parent" + k);
+ par.setProperty("foo", "bar");
+ for (int i = 0; i < NUM_LABELS * 2; i++) {
+ Tree subPar = par.addChild("par" + i);
+ for (int j = 0; j < NUM_LEAF_NODES / (2 * NUM_LABELS); j++) {
+ Tree child = subPar.addChild("c" + j);
+ child.setProperty("cons", "val");
+ int labelNum = rGen.nextInt(NUM_LABELS);
+ child.setProperty("foo", "l" + labelNum);
+ }
+ }
+ }
+
+ /*
+ This test's counter part is MultithreadedOldLuceneFacetProviderReadFailureTest which test old implementation having race condition on index reader opening
+ and closing. The test in MultithreadedOldLuceneFacetProviderReadFailureTest is marked as ignored but can be tested on demand.
+ */
+ @Test
+ public void facet() throws Exception {
+ // Explicitly setting following configs to run DelayedLuceneFacetProvider and a thread sleep of 50 ms in refresh readers. Refer: OAK-8898
+ System.setProperty(LucenePropertyIndex.OLD_FACET_PROVIDER_CONFIG_NAME, "false");
+ // The variable is static final so once set it remains same for all tests and which will lead to slow execution
+ // of other tests as this add a sleep of specified milliseconds in refresh reader method in LuceneIndexNodeManager.
+ // System.setProperty(LuceneIndexNodeManager.OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, "40");
+ Thread.currentThread().setName("main");
+ String idxName = "hybridtest";
+ Tree idx = createIndex(root.getTree("/"), idxName);
+ TestUtil.enableIndexingMode(idx, FulltextIndexConstants.IndexingMode.NRT);
+ setTraversalEnabled(false);
+ root.commit();
+ jcr = new Jcr(oak);
+ jcrRepo = jcr.createRepository();
+ createSmallDataset(0);
+ clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1);
+ root.commit();
+ runAsyncIndex();
+ createSmallDataset(2);
+ clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1);
+ root.commit();
+ Session anonSession = jcrRepo.login(new GuestCredentials());
+ qm = anonSession.getWorkspace().getQueryManager();
+ Query q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2);
+ QueryResult qr = q.execute();
+
+ thread = new Thread(new Runnable() {
+ public void run() {
+ Thread.currentThread().setName("branch");
+ Query q = null;
+ try {
+ clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1);
+ q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2);
+ QueryResult qr = q.execute();
+ } catch (RepositoryException | InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ thread.start();
+ try {
+ RowIterator it = qr.getRows();
+ String firstColumnName = qr.getColumnNames()[0];
+ if (it.hasNext()) {
+ Value v = it.nextRow().getValue(firstColumnName);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ private void runAsyncIndex() {
+ AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, Runnable.class, new Predicate<Runnable>() {
+ @Override
+ public boolean test(@Nullable Runnable input) {
+ return input instanceof AsyncIndexUpdate;
+ }
+ });
+ assertNotNull(async);
+ async.run();
+ if (async.isFailing()) {
+ fail("AsyncIndexUpdate failed");
+ }
+ root.refresh();
+ }
+
+ private Tree createPath(String path) {
+ Tree base = root.getTree("/");
+ for (String e : PathUtils.elements(path)) {
+ base = base.addChild(e);
+ }
+ return base;
+ }
+
+ private Tree createIndex(Tree index, String name) throws RepositoryException {
+ IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
+ idxBuilder.noAsync()
+ .indexRule("nt:base")
+ .property("cons").propertyIndex()
+ .property("foo").propertyIndex()
+ .getBuilderTree().setProperty(PROP_FACETS, true);
+ Tree facetConfig = idxBuilder.getBuilderTree().addChild(FACET_PROP);
+ facetConfig.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
+ facetConfig.setProperty("secure", "statistical");
+ facetConfig.setProperty("topChildren", "100");
+ Tree idxTree = index.getChild("oak:index").addChild(name);
+ idxBuilder.build(idxTree);
+ return idxTree;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/DelayedFacetReadTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java?rev=1875283&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java Tue Mar 17 08:34:04 2020
@@ -0,0 +1,290 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.hybrid;
+
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.ContentRepository;
+import org.apache.jackrabbit.oak.api.Tree;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
+import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.counter.NodeCounterEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexNodeManager;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LucenePropertyIndex;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil;
+import org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.OptionalEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.nodetype.NodeTypeIndexProvider;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.query.AbstractQueryTest;
+import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
+import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
+import org.apache.jackrabbit.oak.spi.security.OpenSecurityProvider;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
+import org.apache.jackrabbit.oak.stats.Clock;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.jetbrains.annotations.Nullable;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import javax.jcr.GuestCredentials;
+import javax.jcr.Repository;
+import javax.jcr.RepositoryException;
+import javax.jcr.Session;
+import javax.jcr.Value;
+import javax.jcr.query.Query;
+import javax.jcr.query.QueryManager;
+import javax.jcr.query.QueryResult;
+import javax.jcr.query.RowIterator;
+import java.io.File;
+import java.io.IOException;
+import java.util.Properties;
+import java.util.Random;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.PROP_FACETS;
+import static org.apache.jackrabbit.oak.plugins.index.search.FulltextIndexConstants.STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+import static org.apache.jackrabbit.oak.spi.mount.Mounts.defaultMountInfoProvider;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
+public class MultithreadedOldLuceneFacetProviderReadFailureTest extends AbstractQueryTest {
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+ private static final int NUM_LABELS = 4;
+ private static final int NUM_LEAF_NODES = STATISTICAL_FACET_SAMPLE_SIZE_DEFAULT;
+ private static final String FACET_PROP = "facets";
+ private static final long REFRESH_DELTA = TimeUnit.SECONDS.toMillis(1);
+
+ private ExecutorService executorService = Executors.newFixedThreadPool(2);
+ private OptionalEditorProvider optionalEditorProvider = new OptionalEditorProvider();
+ private NRTIndexFactory nrtIndexFactory;
+ private LuceneIndexProvider luceneIndexProvider;
+ private NodeStore nodeStore;
+ private DocumentQueue queue;
+ private Clock clock = new Clock.Virtual();
+ private Whiteboard wb;
+ private QueryManager qm;
+ private Repository jcrRepo;
+ private Jcr jcr;
+ private Oak oak;
+ private Thread thread;
+ // backup original system properties i.e. before test started
+ private final Properties backupProperties = (Properties) System.getProperties().clone();
+
+ @After
+ public void tearDown() throws IOException {
+ luceneIndexProvider.close();
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ new ExecutorCloser(executorService).close();
+ nrtIndexFactory.close();
+ // restore original system properties i.e. before test started
+ System.setProperties(backupProperties);
+ }
+
+ @Override
+ protected ContentRepository createRepository() {
+ IndexCopier copier;
+ try {
+ copier = new IndexCopier(executorService, temporaryFolder.getRoot());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ MountInfoProvider mip = defaultMountInfoProvider();
+ nrtIndexFactory = new NRTIndexFactory(copier, clock, TimeUnit.MILLISECONDS.toSeconds(REFRESH_DELTA), StatisticsProvider.NOOP);
+ nrtIndexFactory.setAssertAllResourcesClosed(true);
+ LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
+ IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
+ luceneIndexProvider = new LuceneIndexProvider(tracker);
+ queue = new DocumentQueue(100, tracker, sameThreadExecutor());
+ LuceneIndexEditorProvider editorProvider = new LuceneIndexEditorProvider(copier,
+ tracker,
+ null,
+ null,
+ mip);
+ editorProvider.setIndexingQueue(queue);
+ LocalIndexObserver localIndexObserver = new LocalIndexObserver(queue, StatisticsProvider.NOOP);
+ nodeStore = new MemoryNodeStore();
+ oak = new Oak(nodeStore)
+ .with(new InitialContent())
+ .with(new OpenSecurityProvider())
+ .with((QueryIndexProvider) luceneIndexProvider)
+ .with((Observer) luceneIndexProvider)
+ .with(localIndexObserver)
+ .with(editorProvider)
+ .with(new PropertyIndexEditorProvider())
+ .with(new NodeTypeIndexProvider())
+ .with(optionalEditorProvider)
+ .with(new NodeCounterEditorProvider())
+ //Effectively disable async indexing auto run
+ //such that we can control run timing as per test requirement
+ .withAsyncIndexing("async", TimeUnit.DAYS.toSeconds(1));
+
+ wb = oak.getWhiteboard();
+ ContentRepository repo = oak.createContentRepository();
+ return repo;
+ }
+
+ private void createSmallDataset(int k) throws RepositoryException {
+ Random rGen = new Random(42);
+ Tree par = createPath("/parent" + k);
+ par.setProperty("foo", "bar");
+ for (int i = 0; i < NUM_LABELS * 2; i++) {
+ Tree subPar = par.addChild("par" + i);
+ for (int j = 0; j < NUM_LEAF_NODES / (2 * NUM_LABELS); j++) {
+ Tree child = subPar.addChild("c" + j);
+ child.setProperty("cons", "val");
+ int labelNum = rGen.nextInt(NUM_LABELS);
+ child.setProperty("foo", "l" + labelNum);
+ }
+ }
+ }
+
+ /*
+ The facet test delays threads intentionally so that index reader closes before reading facets. But sometimes it may fail
+ because of load on a system. You can change the system property below between 30-50 and see if this test passes on
+ your system.
+ The following test is ignored as it is not a consistent test and result may vary based on system load etc.
+ This test is a race condition and can be test for failure by using different values for OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME
+ The probable value where this will fail is between 30ms to 60ms.
+ This test's counter part is DelayedFacetReadTest which test new implementation solving the race condition on index reader opening
+ and closing.
+ */
+ @Ignore
+ @Test(expected = RuntimeException.class)
+ public void facet() throws Exception {
+ // Explicitly setting following configs to run LuceneFacetProvider (old) and a thread sleep of 50 ms in refresh readers. Refer: OAK-8898
+ System.setProperty(LucenePropertyIndex.OLD_FACET_PROVIDER_CONFIG_NAME, "true");
+ System.setProperty(LuceneIndexNodeManager.OLD_FACET_PROVIDER_TEST_FAILURE_SLEEP_INSTRUMENT_NAME, "40");
+ Thread.currentThread().setName("main");
+ String idxName = "hybridtest";
+ Tree idx = createIndex(root.getTree("/"), idxName);
+ TestUtil.enableIndexingMode(idx, FulltextIndexConstants.IndexingMode.NRT);
+ setTraversalEnabled(false);
+ root.commit();
+ jcr = new Jcr(oak);
+ jcrRepo = jcr.createRepository();
+ createSmallDataset(0);
+ clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1);
+ root.commit();
+ runAsyncIndex();
+ createSmallDataset(2);
+ clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1);
+ root.commit();
+ Session anonSession = jcrRepo.login(new GuestCredentials());
+ qm = anonSession.getWorkspace().getQueryManager();
+ Query q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2);
+ QueryResult qr = q.execute();
+
+ thread = new Thread(new Runnable() {
+ public void run() {
+ Thread.currentThread().setName("branch");
+ Query q = null;
+ try {
+ clock.waitUntil(clock.getTime() + REFRESH_DELTA + 1);
+ q = qm.createQuery("SELECT [rep:facet(foo)] FROM [nt:base] WHERE [cons] = 'val'", SQL2);
+ QueryResult qr = q.execute();
+ } catch (RepositoryException | InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ thread.start();
+ RowIterator it;
+ try {
+ it = qr.getRows();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("org.apache.lucene.store.AlreadyClosedException: this IndexReader is closed"));
+ e.printStackTrace(System.out);
+ throw e;
+ }
+ String firstColumnName = qr.getColumnNames()[0];
+ if (it.hasNext()) {
+ Value v = it.nextRow().getValue(firstColumnName);
+ }
+ }
+
+ private void runAsyncIndex() {
+ AsyncIndexUpdate async = (AsyncIndexUpdate) WhiteboardUtils.getService(wb, Runnable.class, new Predicate<Runnable>() {
+ @Override
+ public boolean test(@Nullable Runnable input) {
+ return input instanceof AsyncIndexUpdate;
+ }
+ });
+ assertNotNull(async);
+ async.run();
+ if (async.isFailing()) {
+ fail("AsyncIndexUpdate failed");
+ }
+ root.refresh();
+ }
+
+ private Tree createPath(String path) {
+ Tree base = root.getTree("/");
+ for (String e : PathUtils.elements(path)) {
+ base = base.addChild(e);
+ }
+ return base;
+ }
+
+ private Tree createIndex(Tree index, String name) throws RepositoryException {
+ IndexDefinitionBuilder idxBuilder = new IndexDefinitionBuilder();
+ idxBuilder.noAsync()
+ .indexRule("nt:base")
+ .property("cons").propertyIndex()
+ .property("foo").propertyIndex()
+ .getBuilderTree().setProperty(PROP_FACETS, true);
+ Tree facetConfig = idxBuilder.getBuilderTree().addChild(FACET_PROP);
+ facetConfig.setProperty("jcr:primaryType", "nt:unstructured", Type.NAME);
+ facetConfig.setProperty("secure", "statistical");
+ facetConfig.setProperty("topChildren", "100");
+ Tree idxTree = index.getChild("oak:index").addChild(name);
+ idxBuilder.build(idxTree);
+ return idxTree;
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/MultithreadedOldLuceneFacetProviderReadFailureTest.java
------------------------------------------------------------------------------
svn:eol-style = native