You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/10/03 05:10:21 UTC
svn commit: r1810648 -
/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
Author: chetanm
Date: Tue Oct 3 05:10:21 2017
New Revision: 1810648
URL: http://svn.apache.org/viewvc?rev=1810648&view=rev
Log:
OAK-6535 - Synchronous Lucene Property Indexes
Update the HybridIndex benchmark to support sync indexes
-- `syncIndexing` system property to enable sync indexing based on
Hybrid Index v2
-- `cleanerIntervalInSecs` - Specifies the time between 2 runs of cleaner
Defaults to 10 secs for benchmark
-- Now the `nodetype` index would be configured with explicit list
Modified:
jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
Modified: jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java?rev=1810648&r1=1810647&r2=1810648&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java (original)
+++ jackrabbit/oak/trunk/oak-benchmarks/src/main/java/org/apache/jackrabbit/oak/benchmark/HybridIndexTest.java Tue Oct 3 05:10:21 2017
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.benchm
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -46,15 +47,22 @@ import javax.jcr.query.QueryResult;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterators;
+import com.google.common.collect.Lists;
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.Oak;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.api.jmx.IndexStatsMBean;
import org.apache.jackrabbit.oak.fixture.JcrCreator;
import org.apache.jackrabbit.oak.fixture.OakRepositoryFixture;
import org.apache.jackrabbit.oak.fixture.RepositoryFixture;
import org.apache.jackrabbit.oak.jcr.Jcr;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoService;
+import org.apache.jackrabbit.oak.plugins.index.AsyncIndexInfoServiceImpl;
import org.apache.jackrabbit.oak.plugins.index.AsyncIndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathService;
+import org.apache.jackrabbit.oak.plugins.index.IndexPathServiceImpl;
import org.apache.jackrabbit.oak.plugins.index.IndexUtils;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexTracker;
@@ -64,15 +72,19 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.DocumentQueue;
import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.LocalIndexObserver;
import org.apache.jackrabbit.oak.plugins.index.lucene.hybrid.NRTIndexFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.property.PropertyIndexCleaner;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.DefaultIndexReaderFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.reader.LuceneIndexReaderFactory;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
-import org.apache.jackrabbit.oak.spi.commit.Observer;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder.PropertyRule;
+import org.apache.jackrabbit.oak.spi.commit.BackgroundObserver;
import org.apache.jackrabbit.oak.spi.lifecycle.RepositoryInitializer;
import org.apache.jackrabbit.oak.spi.mount.MountInfoProvider;
import org.apache.jackrabbit.oak.spi.mount.Mounts;
import org.apache.jackrabbit.oak.spi.query.QueryIndexProvider;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.spi.whiteboard.Registration;
import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
import org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils;
import org.apache.jackrabbit.oak.stats.Clock;
@@ -82,7 +94,10 @@ import org.slf4j.LoggerFactory;
import static com.google.common.base.Preconditions.checkNotNull;
import static java.util.Collections.singleton;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.DECLARING_NODE_TYPES;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.INDEX_DEFINITIONS_NODE_TYPE;
import static org.apache.jackrabbit.oak.spi.nodetype.NodeTypeConstants.NT_OAK_UNSTRUCTURED;
+import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.scheduleWithFixedDelay;
public class HybridIndexTest extends AbstractTest<HybridIndexTest.TestContext> {
enum Status {
@@ -113,10 +128,12 @@ public class HybridIndexTest extends Abs
private int numOfIndexes = Integer.getInteger("numOfIndexes", 10);
private int refreshDeltaMillis = Integer.getInteger("refreshDeltaMillis", 1000);
private int asyncInterval = Integer.getInteger("asyncInterval", 5);
+ private int cleanerIntervalInSecs = Integer.getInteger("cleanerIntervalInSecs", 10);
private int queueSize = Integer.getInteger("queueSize", 1000);
private boolean hybridIndexEnabled = Boolean.getBoolean("hybridIndexEnabled");
private boolean dumpStats = Boolean.getBoolean("dumpStats");
private boolean useOakCodec = Boolean.parseBoolean(System.getProperty("useOakCodec", "true"));
+ private boolean syncIndexing = Boolean.parseBoolean(System.getProperty("syncIndexing", "false"));
private String indexingMode = System.getProperty("indexingMode", "nrt");
private boolean searcherEnabled = Boolean.parseBoolean(System.getProperty("searcherEnabled", "true"));
@@ -139,6 +156,8 @@ public class HybridIndexTest extends Abs
private final Logger log = LoggerFactory.getLogger(getClass());
private final ExecutorService executorService = MoreExecutors.getExitingExecutorService(
(ThreadPoolExecutor) Executors.newFixedThreadPool(5));
+ private final List<Registration> regs = new ArrayList<>();
+ private BackgroundObserver backgroundObserver;
public HybridIndexTest(File workDir, StatisticsProvider statsProvider) {
@@ -153,17 +172,21 @@ public class HybridIndexTest extends Abs
@Override
public Jcr customize(Oak oak) {
Jcr jcr = new Jcr(oak);
- prepareLuceneIndexer(workDir);
+ whiteboard = oak.getWhiteboard();
+ prepareLuceneIndexer(workDir, getNodeStore(oak));
+
+ backgroundObserver = new BackgroundObserver(luceneIndexProvider, executorService, 5);
+
jcr.with((QueryIndexProvider) luceneIndexProvider)
- .with((Observer) luceneIndexProvider)
- .with(luceneEditorProvider);
+ .with(backgroundObserver)
+ .with(luceneEditorProvider)
+ .with(new NodeTypeIndexFixerInitializer());
if (hybridIndexEnabled) {
jcr.with(localIndexObserver);
indexInitializer = new LuceneIndexInitializer();
}
- whiteboard = oak.getWhiteboard();
jcr.with(indexInitializer);
//Configure the default global fulltext index as it impacts
@@ -238,6 +261,14 @@ public class HybridIndexTest extends Abs
//and before NodeStore teardown
getAsyncIndexUpdate().close();
+ if (backgroundObserver != null){
+ backgroundObserver.close();
+ }
+
+ for (Registration r : regs) {
+ r.unregister();
+ }
+
//Close hybrid stuff after async is closed
if (hybridIndexEnabled){
queue.close();
@@ -248,9 +279,10 @@ public class HybridIndexTest extends Abs
FileUtils.deleteDirectory(indexCopierDir);
}
System.out.printf("numOfIndexes: %d, refreshDeltaMillis: %d, asyncInterval: %d, queueSize: %d , " +
- "hybridIndexEnabled: %s, indexingMode: %s, useOakCodec: %s %n",
+ "hybridIndexEnabled: %s, indexingMode: %s, useOakCodec: %s, cleanerIntervalInSecs: %d, " +
+ "syncIndexing: %s %n",
numOfIndexes, refreshDeltaMillis, asyncInterval, queueSize, hybridIndexEnabled,
- indexingMode, useOakCodec);
+ indexingMode, useOakCodec, cleanerIntervalInSecs, syncIndexing);
if (dumpStats) {
dumpStats();
@@ -281,6 +313,9 @@ public class HybridIndexTest extends Abs
if (useOakCodec){
commentElements.add("oakCodec");
}
+ if (syncIndexing) {
+ commentElements.add("sync");
+ }
} else {
commentElements.add("property");
}
@@ -319,7 +354,7 @@ public class HybridIndexTest extends Abs
return status.name();
}
- private void prepareLuceneIndexer(File workDir) {
+ private void prepareLuceneIndexer(File workDir, NodeStore nodeStore) {
try {
indexCopierDir = createTemporaryFolderIn(workDir);
copier = new IndexCopier(executorService, indexCopierDir, true);
@@ -327,11 +362,16 @@ public class HybridIndexTest extends Abs
throw new RuntimeException(e);
}
+ IndexPathService indexPathService = new IndexPathServiceImpl(nodeStore);
+ AsyncIndexInfoService asyncIndexInfoService = new AsyncIndexInfoServiceImpl(nodeStore);
+
nrtIndexFactory = new NRTIndexFactory(copier, Clock.SIMPLE,
TimeUnit.MILLISECONDS.toSeconds(refreshDeltaMillis), StatisticsProvider.NOOP);
MountInfoProvider mip = Mounts.defaultMountInfoProvider();
LuceneIndexReaderFactory indexReaderFactory = new DefaultIndexReaderFactory(mip, copier);
+
IndexTracker tracker = new IndexTracker(indexReaderFactory, nrtIndexFactory);
+
luceneIndexProvider = new LuceneIndexProvider(tracker);
luceneEditorProvider = new LuceneIndexEditorProvider(copier,
tracker,
@@ -343,6 +383,13 @@ public class HybridIndexTest extends Abs
localIndexObserver = new LocalIndexObserver(queue, statsProvider);
luceneEditorProvider.setIndexingQueue(queue);
+ if (syncIndexing) {
+ PropertyIndexCleaner cleaner = new PropertyIndexCleaner(nodeStore, indexPathService, asyncIndexInfoService);
+ regs.add(scheduleWithFixedDelay(whiteboard, cleaner,
+ cleanerIntervalInSecs, true, true));
+ }
+
+
Thread.setDefaultUncaughtExceptionHandler((t, e) -> log.warn("Uncaught exception", e));
}
@@ -375,6 +422,16 @@ public class HybridIndexTest extends Abs
return createdFolder;
}
+ private static NodeStore getNodeStore(Oak oak) {
+ try {
+ Field f = Oak.class.getDeclaredField("store");
+ f.setAccessible(true);
+ return (NodeStore) f.get(oak);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
private class PropertyIndexInitializer implements RepositoryInitializer {
@Override
@@ -405,7 +462,10 @@ public class HybridIndexTest extends Abs
IndexDefinitionBuilder defnBuilder = new IndexDefinitionBuilder();
defnBuilder.evaluatePathRestrictions();
defnBuilder.async("async", indexingMode, "async");
- defnBuilder.indexRule("nt:base").property(indexedPropName).propertyIndex();
+ PropertyRule pr = defnBuilder.indexRule("nt:base").property(indexedPropName).propertyIndex();
+ if (syncIndexing) {
+ pr.sync();
+ }
if (useOakCodec) {
defnBuilder.codec("oakCodec");
}
@@ -433,6 +493,29 @@ public class HybridIndexTest extends Abs
}
}
+ private class NodeTypeIndexFixerInitializer implements RepositoryInitializer {
+
+ @Override
+ public void initialize(@Nonnull NodeBuilder builder) {
+ //Due to OAK-1150 currently all nodes get indexed
+ //With explicit list on those nodes would be indexed
+ NodeBuilder nodetype = builder.getChildNode("oak:index").getChildNode("nodetype");
+ if (nodetype.exists()) {
+ List<String> nodetypes = Lists.newArrayList();
+ if (nodetype.hasProperty(DECLARING_NODE_TYPES)){
+ nodetypes = Lists.newArrayList(nodetype.getProperty(DECLARING_NODE_TYPES).getValue(Type.STRINGS));
+ }
+
+ if (nodetypes.isEmpty()) {
+ nodetypes.add(INDEX_DEFINITIONS_NODE_TYPE);
+ nodetypes.add("rep:Authorizable");
+ nodetype.setProperty(DECLARING_NODE_TYPES, nodetypes, Type.NAMES);
+ nodetype.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+ }
+ }
+ }
+ }
+
private class Searcher implements Runnable {
final Session session = loginWriter();
int resultSize = 0;