You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/05/22 11:00:31 UTC
svn commit: r1795791 - in /jackrabbit/oak/trunk/oak-run/src:
main/java/org/apache/jackrabbit/oak/index/ main/resources/
test/java/org/apache/jackrabbit/oak/index/
Author: chetanm
Date: Mon May 22 11:00:31 2017
New Revision: 1795791
URL: http://svn.apache.org/viewvc?rev=1795791&view=rev
Log:
OAK-6246 - Support for out of band indexing with read only access to NodeStore
First part implementation which allows storing index locally and
connects to repository in read only mode.
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java (with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java?rev=1795791&r1=1795790&r2=1795791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java Mon May 22 11:00:31 2017
@@ -35,6 +35,8 @@ import org.apache.jackrabbit.oak.run.com
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import static com.google.common.base.Preconditions.checkNotNull;
+
public class IndexCommand implements Command {
public static final String NAME = "index";
public static final String INDEX_DEFINITIONS_JSON = "index-definitions.json";
@@ -46,12 +48,13 @@ public class IndexCommand implements Com
private File info;
private File definitions;
private File consistencyCheckReport;
+ private Options opts;
@Override
public void execute(String... args) throws Exception {
OptionParser parser = new OptionParser();
- Options opts = new Options();
+ opts = new Options();
opts.setCommandName(NAME);
opts.setSummary(summary);
opts.setConnectionString(CommonOptions.DEFAULT_CONNECTION_STRING);
@@ -100,7 +103,16 @@ public class IndexCommand implements Com
if (!indexOpts.isReindex()){
return;
}
- new ReIndexer(indexHelper).reindex();
+
+ if (opts.getCommonOpts().isReadWrite()) {
+ new ReIndexer(indexHelper).reindex();
+ } else {
+ String checkpoint = indexOpts.getCheckpoint();
+ checkNotNull(checkpoint, "Checkpoint value is required for reindexing done in read only mode");
+ try (OutOfBandIndexer indexer = new OutOfBandIndexer(indexHelper, checkpoint)) {
+ indexer.reindex();
+ }
+ }
}
private void dumpIndexContents(IndexOptions indexOpts, IndexHelper indexHelper) throws IOException {
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java?rev=1795791&r1=1795790&r2=1795791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexOptions.java Mon May 22 11:00:31 2017
@@ -56,6 +56,7 @@ public class IndexOptions implements Opt
private OptionSet options;
private final Set<OptionSpec> actionOpts;
private final OptionSpec<String> indexPaths;
+ private final OptionSpec<String> checkpoint;
private final Set<String> operationNames;
@@ -71,6 +72,10 @@ public class IndexOptions implements Opt
"selected operations need to be performed")
.withRequiredArg().ofType(String.class).withValuesSeparatedBy(",");
+ checkpoint = parser.accepts("checkpoint", "Checkpoint value upto which index would be updated when " +
+ "indexing is performed in read only mode")
+ .withRequiredArg().ofType(String.class);
+
consistencyCheck = parser.accepts("index-consistency-check", "Performs consistency check " +
"for indexes as specified by --index-paths. If none specified performs check for all indexes. Currently " +
"this is only supported for Lucene indexes. Possible values 1 - Basic check, 2 - Full check (slower)")
@@ -146,6 +151,10 @@ public class IndexOptions implements Opt
return options.has(reindex);
}
+ public String getCheckpoint(){
+ return checkpoint.value(options);
+ }
+
public List<String> getIndexPaths(){
return options.has(indexPaths) ? trim(indexPaths.values(options)) : Collections.emptyList();
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java?rev=1795791&r1=1795790&r2=1795791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/LuceneIndexHelper.java Mon May 22 11:00:31 2017
@@ -28,13 +28,16 @@ import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.plugins.index.lucene.ExtractedTextCache;
import org.apache.jackrabbit.oak.plugins.index.lucene.IndexCopier;
import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
class LuceneIndexHelper implements Closeable {
private final IndexHelper indexHelper;
private IndexCopier indexCopier;
+ //TODO Set pre extracted text provider
private final ExtractedTextCache textCache =
new ExtractedTextCache(FileUtils.ONE_MB * 5, TimeUnit.HOURS.toSeconds(5));
+ private DirectoryFactory directoryFactory;
LuceneIndexHelper(IndexHelper indexHelper) {
this.indexHelper = indexHelper;
@@ -52,9 +55,17 @@ class LuceneIndexHelper implements Close
editor.setBlobStore((GarbageCollectableBlobStore) indexHelper.getBlobStore());
}
+ if (directoryFactory != null) {
+ editor.setDirectoryFactory(directoryFactory);
+ }
+
return editor;
}
+ public void setDirectoryFactory(DirectoryFactory directoryFactory) {
+ this.directoryFactory = directoryFactory;
+ }
+
private IndexCopier getIndexCopier() throws IOException {
if (indexCopier == null) {
File indexWorkDir = new File(indexHelper.getWorkDir(), "indexWorkDir");
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java?rev=1795791&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java Mon May 22 11:00:31 2017
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.index;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Properties;
+
+import com.google.common.base.Stopwatch;
+import com.google.common.io.Closer;
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.index.CompositeIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.CorruptIndexHandler;
+import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.IndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdate;
+import org.apache.jackrabbit.oak.plugins.index.IndexUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.index.NodeTraversalCallback;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.DirectoryFactory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.FSDirectoryFactory;
+import org.apache.jackrabbit.oak.plugins.index.property.PropertyIndexEditorProvider;
+import org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState;
+import org.apache.jackrabbit.oak.plugins.memory.MemoryNodeStore;
+import org.apache.jackrabbit.oak.plugins.memory.PropertyStates;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EditorDiff;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.commit.VisibleEditor;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.Arrays.asList;
+import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.ASYNC_PROPERTY_NAME;
+
+public class OutOfBandIndexer implements Closeable, IndexUpdateCallback, NodeTraversalCallback {
+ private final Logger log = LoggerFactory.getLogger(getClass());
+ /**
+ * Index lane name which is used for indexing
+ */
+ private static final String REINDEX_LANE = "offline-reindex-async";
+ /**
+ * Property name where previous value of 'async' is stored
+ */
+ private static final String ASYNC_PREVIOUS = "async-previous";
+ /**
+ * Value stored in previous async property if the index is not async
+ * i.e. when a sync index is reindexed in out of band mode
+ */
+ private static final String ASYNC_PREVIOUS_NONE = "none";
+ /**
+ * Directory name in output directory under which indexes are
+ * stored
+ */
+ public static final String LOCAL_INDEX_ROOT_DIR = "indexes";
+ /**
+ * File name stored in final index directory which contains meta
+ * information like checkpoint details. This can be used by
+ * importer while importing the indexes
+ */
+ private static final String INDEXER_META = "indexer-info.txt";
+
+ private final Closer closer = Closer.create();
+ private final IndexHelper indexHelper;
+ private final String checkpoint;
+ private Map<String, String> checkpointInfo = Collections.emptyMap();
+ private NodeStore copyOnWriteStore;
+ private LuceneIndexHelper luceneIndexHelper;
+ private File localIndexDir;
+
+ //TODO Support for providing custom index definition i.e. where definition is not
+ //present in target repository
+
+ public OutOfBandIndexer(IndexHelper indexHelper, String checkpoint) {
+ this.indexHelper = checkNotNull(indexHelper);
+ this.checkpoint = checkNotNull(checkpoint);
+ }
+
+ public void reindex() throws CommitFailedException, IOException {
+ Stopwatch w = Stopwatch.createStarted();
+ NodeState checkpointedState = indexHelper.getNodeStore().retrieve(checkpoint);
+
+ checkNotNull(checkpointedState, "Not able to retrieve revision referred via checkpoint [%s]", checkpoint);
+ copyOnWriteStore = new MemoryNodeStore(checkpointedState);
+ checkpointInfo = indexHelper.getNodeStore().checkpointInfo(checkpoint);
+ //TODO Check for indexPaths being empty
+
+ log.info("Proceeding to index {} upto checkpoint {} {}", indexHelper.getIndexPaths(), checkpoint, checkpointInfo);
+
+ switchIndexLanesAndReindexFlag();
+ preformIndexUpdate();
+ writeMetaInfo();
+ copyIndexFilesToOutput();
+
+ log.info("Indexing completed for indexes {} in {} and index files are copied to {}",
+ indexHelper.getIndexPaths(), w, IndexCommand.getPath(indexHelper.getOutputDir()));
+ }
+
+ private File getLocalIndexDir() throws IOException {
+ if (localIndexDir == null) {
+ localIndexDir = new File(indexHelper.getWorkDir(), LOCAL_INDEX_ROOT_DIR);
+ FileUtils.forceMkdir(localIndexDir);
+ }
+ return localIndexDir;
+ }
+
+ @Override
+ public void close() throws IOException {
+ closer.close();
+ }
+
+ //~---------------------------------------------------< callbacks >
+
+ @Override
+ public void indexUpdate() throws CommitFailedException {
+
+ }
+
+ @Override
+ public void traversedNode() throws CommitFailedException {
+
+ }
+
+ private void preformIndexUpdate() throws IOException, CommitFailedException {
+ NodeBuilder builder = copyOnWriteStore.getRoot().builder();
+
+ IndexUpdate indexUpdate = new IndexUpdate(
+ createIndexEditorProvider(),
+ REINDEX_LANE,
+ copyOnWriteStore.getRoot(),
+ builder,
+ this,
+ this,
+ CommitInfo.EMPTY,
+ CorruptIndexHandler.NOOP
+ );
+
+ NodeState before = EmptyNodeState.EMPTY_NODE;
+ NodeState after = copyOnWriteStore.getRoot();
+
+ CommitFailedException exception =
+ EditorDiff.process(VisibleEditor.wrap(indexUpdate), before, after);
+
+ if (exception != null) {
+ throw exception;
+ }
+ }
+
+ private IndexEditorProvider createIndexEditorProvider() throws IOException {
+ IndexEditorProvider lucene = createLuceneEditorProvider();
+ IndexEditorProvider property = new PropertyIndexEditorProvider().with(indexHelper.getMountInfoProvider());
+
+ return CompositeIndexEditorProvider.compose(asList(lucene, property));
+ }
+
+ private IndexEditorProvider createLuceneEditorProvider() throws IOException {
+ luceneIndexHelper = new LuceneIndexHelper(indexHelper);
+ DirectoryFactory dirFactory = new FSDirectoryFactory(getLocalIndexDir());
+ luceneIndexHelper.setDirectoryFactory(dirFactory);
+ closer.register(luceneIndexHelper);
+ return luceneIndexHelper.createEditorProvider();
+ }
+
+ private void switchIndexLanesAndReindexFlag() throws CommitFailedException {
+ NodeBuilder builder = copyOnWriteStore.getRoot().builder();
+
+ for (String indexPath : indexHelper.getIndexPaths()) {
+ NodeBuilder idxBuilder = NodeStoreUtils.childBuilder(builder, indexPath);
+ idxBuilder.setProperty(IndexConstants.REINDEX_PROPERTY_NAME, true);
+ switchLane(idxBuilder);
+ }
+
+ copyOnWriteStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ log.info("Switched the async lane for indexes at {} to {} and marked them for reindex", indexHelper.getIndexPaths(), REINDEX_LANE);
+ }
+
+ /**
+ * Make a copy of current async value and replace it with one required for offline reindexing
+ */
+ static void switchLane(NodeBuilder idxBuilder) {
+ PropertyState currentAsyncState = idxBuilder.getProperty(ASYNC_PROPERTY_NAME);
+ PropertyState newAsyncState = PropertyStates.createProperty(ASYNC_PROPERTY_NAME, REINDEX_LANE, Type.STRING);
+
+ PropertyState previousAsyncState;
+ if (currentAsyncState == null) {
+ previousAsyncState = PropertyStates.createProperty(ASYNC_PREVIOUS, ASYNC_PREVIOUS_NONE);
+ } else {
+ //Ensure that previous state is copied with correct type
+ if (currentAsyncState.isArray()) {
+ previousAsyncState = PropertyStates.createProperty(ASYNC_PREVIOUS, currentAsyncState.getValue(Type.STRINGS), Type.STRINGS);
+ } else {
+ previousAsyncState = PropertyStates.createProperty(ASYNC_PREVIOUS, currentAsyncState.getValue(Type.STRING), Type.STRING);
+ }
+ }
+
+ idxBuilder.setProperty(previousAsyncState);
+ idxBuilder.setProperty(newAsyncState);
+ }
+
+ private void writeMetaInfo() throws IOException {
+ Properties props = new Properties();
+ props.put("checkpoint", checkpoint);
+ try (OutputStream os = FileUtils.openOutputStream(new File(getLocalIndexDir(), INDEXER_META))) {
+ props.store(os, "Indexer info");
+ }
+ }
+
+ private void copyIndexFilesToOutput() throws IOException {
+ FileUtils.moveDirectoryToDirectory(getLocalIndexDir(), indexHelper.getOutputDir(), true);
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/OutOfBandIndexer.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml?rev=1795791&r1=1795790&r2=1795791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/resources/logback.xml Mon May 22 11:00:31 2017
@@ -41,6 +41,9 @@
<!-- Metrics Reporting-->
<logger name="org.apache.jackrabbit.oak.metrics" level="INFO"/>
+ <!-- For index tooling -->
+ <logger name="org.apache.jackrabbit.oak.index" level="INFO"/>
+
<!-- Display information messages from oak-run check (oak-segment) -->
<logger name="org.apache.jackrabbit.oak.plugins.segment.file.tooling.ConsistencyChecker" level="DEBUG"/>
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java?rev=1795791&r1=1795790&r2=1795791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java Mon May 22 11:00:31 2017
@@ -21,6 +21,8 @@ package org.apache.jackrabbit.oak.index;
import java.io.File;
import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
import javax.jcr.Node;
import javax.jcr.PropertyType;
@@ -31,6 +33,8 @@ import com.google.common.io.Files;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.index.IndexConstants;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexRootDirectory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.directory.LocalIndexDir;
import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.After;
@@ -197,6 +201,46 @@ public class IndexCommandIT {
assertEquals(2, reindexCount.getValue(Type.LONG).longValue());
}
+ @Test
+ public void reindexOutOfBand() throws Exception{
+ createTestData(true);
+ fixture.getAsyncIndexUpdate("async").run();
+
+ String checkpoint = fixture.getNodeStore().checkpoint(TimeUnit.HOURS.toMillis(24));
+
+ //Close the repository so as all changes are flushed
+ fixture.close();
+
+ IndexCommand command = new IndexCommand();
+
+ File outDir = temporaryFolder.newFolder();
+ File storeDir = fixture.getDir();
+ String[] args = {
+ "--index-temp-dir=" + temporaryFolder.newFolder().getAbsolutePath(),
+ "--index-out-dir=" + outDir.getAbsolutePath(),
+ "--index-paths=/oak:index/fooIndex",
+ "--checkpoint="+checkpoint,
+ "--reindex",
+ "--", // -- indicates that options have ended and rest needs to be treated as non option
+ storeDir.getAbsolutePath()
+ };
+
+ command.execute(args);
+
+ RepositoryFixture fixture2 = new RepositoryFixture(storeDir);
+ NodeStore store2 = fixture2.getNodeStore();
+ PropertyState reindexCount = getNode(store2.getRoot(), "/oak:index/fooIndex").getProperty(IndexConstants.REINDEX_COUNT);
+ assertEquals(1, reindexCount.getValue(Type.LONG).longValue());
+
+ File indexes = new File(outDir, OutOfBandIndexer.LOCAL_INDEX_ROOT_DIR);
+ assertTrue(indexes.exists());
+
+ IndexRootDirectory idxRoot = new IndexRootDirectory(indexes);
+ List<LocalIndexDir> idxDirs = idxRoot.getAllLocalIndexes();
+
+ assertEquals(1, idxDirs.size());
+ }
+
private void createTestData(boolean asyncIndex) throws IOException, RepositoryException {
fixture = new RepositoryFixture(temporaryFolder.newFolder());
indexIndexDefinitions();