You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/07/10 12:00:15 UTC
svn commit: r1801451 - in /jackrabbit/oak/trunk/oak-run/src:
main/java/org/apache/jackrabbit/oak/index/
test/java/org/apache/jackrabbit/oak/index/
Author: chetanm
Date: Mon Jul 10 12:00:15 2017
New Revision: 1801451
URL: http://svn.apache.org/viewvc?rev=1801451&view=rev
Log:
OAK-6271 - Support for importing index files
Now the IndexCommand when run with --read-write mode and --reindex
would perform following
1. Connect to repo in rw mode and create checkpoint and disconnect
2. Connect again to repo in read only mode and perform reindexing
3. Connect again to repo in rw mode and import the index
This ensures that oak-run is connected to repo in rw mode for minimal
time and hence minimal chances of any side affect
Added:
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/ReindexIT.java (contents, props changed)
- copied, changed from r1801449, jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexImportIT.java
Removed:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/ReIndexer.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/SimpleAsyncReindexer.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexImportIT.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java?rev=1801451&r1=1801450&r2=1801451&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/index/IndexCommand.java Mon Jul 10 12:00:15 2017
@@ -22,20 +22,20 @@ package org.apache.jackrabbit.oak.index;
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.concurrent.TimeUnit;
+import com.google.common.collect.ImmutableMap;
import com.google.common.io.Closer;
import joptsimple.OptionParser;
import org.apache.commons.io.FileUtils;
import org.apache.felix.inventory.Format;
import org.apache.jackrabbit.oak.api.CommitFailedException;
-import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
import org.apache.jackrabbit.oak.run.cli.CommonOptions;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixture;
import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
import org.apache.jackrabbit.oak.run.cli.Options;
import org.apache.jackrabbit.oak.run.commons.Command;
-import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -72,11 +72,15 @@ public class IndexCommand implements Com
//directory might be used by NodeStore for cache stuff like persistentCache
setupDirectories(indexOpts);
- NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts);
- try (Closer closer = Closer.create()) {
- closer.register(fixture);
- execute(fixture.getStore(), fixture.getBlobStore(), fixture.getWhiteboard(), indexOpts, closer);
- tellReportPaths();
+ if (indexOpts.isReindex() && opts.getCommonOpts().isReadWrite()) {
+ performReindexInReadWriteMode(indexOpts);
+ } else {
+ try (Closer closer = Closer.create()) {
+ NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts);
+ closer.register(fixture);
+ execute(fixture, indexOpts, closer);
+ tellReportPaths();
+ }
}
}
@@ -94,14 +98,9 @@ public class IndexCommand implements Com
}
}
- private void execute(NodeStore store, BlobStore blobStore, Whiteboard whiteboard,
- IndexOptions indexOpts, Closer closer) throws IOException, CommitFailedException {
- IndexHelper indexHelper = new IndexHelper(store, blobStore, whiteboard, indexOpts.getOutDir(),
- indexOpts.getWorkDir(), indexOpts.getIndexPaths());
-
- configurePreExtractionSupport(indexOpts, indexHelper);
-
- closer.register(indexHelper);
+ private void execute(NodeStoreFixture fixture, IndexOptions indexOpts, Closer closer)
+ throws IOException, CommitFailedException {
+ IndexHelper indexHelper = createIndexHelper(fixture, indexOpts, closer);
dumpIndexStats(indexOpts, indexHelper);
dumpIndexDefinitions(indexOpts, indexHelper);
@@ -111,6 +110,18 @@ public class IndexCommand implements Com
importIndex(indexOpts, indexHelper);
}
+ @NotNull
+ private IndexHelper createIndexHelper(NodeStoreFixture fixture,
+ IndexOptions indexOpts, Closer closer) throws IOException {
+ IndexHelper indexHelper = new IndexHelper(fixture.getStore(), fixture.getBlobStore(), fixture.getWhiteboard(),
+ indexOpts.getOutDir(), indexOpts.getWorkDir(), indexOpts.getIndexPaths());
+
+ configurePreExtractionSupport(indexOpts, indexHelper);
+
+ closer.register(indexHelper);
+ return indexHelper;
+ }
+
private void configurePreExtractionSupport(IndexOptions indexOpts, IndexHelper indexHelper) throws IOException {
File preExtractedTextDir = indexOpts.getPreExtractedTextDir();
if (preExtractedTextDir != null) {
@@ -124,21 +135,65 @@ public class IndexCommand implements Com
return;
}
- if (opts.getCommonOpts().isReadWrite()) {
- new ReIndexer(indexHelper).reindex();
- } else {
- String checkpoint = indexOpts.getCheckpoint();
- checkNotNull(checkpoint, "Checkpoint value is required for reindexing done in read only mode");
- try (OutOfBandIndexer indexer = new OutOfBandIndexer(indexHelper, checkpoint)) {
- indexer.reindex();
- }
+ String checkpoint = indexOpts.getCheckpoint();
+ reindex(indexHelper, checkpoint);
+ }
+
+ private void reindex(IndexHelper indexHelper, String checkpoint) throws IOException, CommitFailedException {
+ checkNotNull(checkpoint, "Checkpoint value is required for reindexing done in read only mode");
+ try (OutOfBandIndexer indexer = new OutOfBandIndexer(indexHelper, checkpoint)) {
+ indexer.reindex();
}
}
private void importIndex(IndexOptions indexOpts, IndexHelper indexHelper) throws IOException, CommitFailedException {
if (indexOpts.isImportIndex()) {
File importDir = indexOpts.getIndexImportDir();
- new IndexImporterSupport(indexHelper).importIndex(importDir);
+ importIndex(indexHelper, importDir);
+ }
+ }
+
+ private void importIndex(IndexHelper indexHelper, File importDir) throws IOException, CommitFailedException {
+ new IndexImporterSupport(indexHelper).importIndex(importDir);
+ }
+
+ private void performReindexInReadWriteMode(IndexOptions indexOpts) throws Exception {
+ //TODO To support restart we need to store this checkpoint somewhere
+ String checkpoint = connectInReadWriteModeAndCreateCheckPoint(indexOpts);
+ log.info("Created checkpoint [{}] for indexing", checkpoint);
+
+ log.info("Proceeding to reindex with read only access to NodeStore");
+ File indexDir = performReindexInReadOnlyMode(indexOpts, checkpoint);
+
+ log.info("Proceeding to import index data from [{}] by connecting to NodeStore in read-write mode", getPath(indexDir));
+ connectInReadWriteModeAndImportIndex(indexOpts, indexDir);
+
+ log.info("Indexes imported successfully");
+ }
+
+ private File performReindexInReadOnlyMode(IndexOptions indexOpts, String checkpoint) throws Exception {
+ try (Closer closer = Closer.create()) {
+ NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts, true);
+ closer.register(fixture);
+ IndexHelper indexHelper = createIndexHelper(fixture, indexOpts, closer);
+ reindex(indexHelper, checkpoint);
+ return new File(indexOpts.getOutDir(), OutOfBandIndexer.LOCAL_INDEX_ROOT_DIR);
+ }
+ }
+
+ private String connectInReadWriteModeAndCreateCheckPoint(IndexOptions indexOpts) throws Exception {
+ try (NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts)) {
+ return fixture.getStore().checkpoint(TimeUnit.DAYS.toMillis(100),
+ ImmutableMap.of("creator", "oak-run-indexer"));
+ }
+ }
+
+ private void connectInReadWriteModeAndImportIndex(IndexOptions indexOpts, File indexDir) throws Exception {
+ try (Closer closer = Closer.create()) {
+ NodeStoreFixture fixture = NodeStoreFixtureProvider.create(opts);
+ closer.register(fixture);
+ IndexHelper indexHelper = createIndexHelper(fixture, indexOpts, closer);
+ importIndex(indexHelper, indexDir);
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java?rev=1801451&r1=1801450&r2=1801451&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexCommandIT.java Mon Jul 10 12:00:15 2017
@@ -150,36 +150,4 @@ public class IndexCommandIT extends Abst
File dumpDir = new File(outDir, IndexDumper.INDEX_DUMPS_DIR);
assertTrue(dumpDir.exists());
}
-
- @Test
- public void reindex() throws Exception{
- createTestData(true);
- fixture.getAsyncIndexUpdate("async").run();
- //Close the repository so as all changes are flushed
- fixture.close();
-
- IndexCommand command = new IndexCommand();
-
- File outDir = temporaryFolder.newFolder();
- File storeDir = fixture.getDir();
- String[] args = {
- "--index-temp-dir=" + temporaryFolder.newFolder().getAbsolutePath(),
- "--index-out-dir=" + outDir.getAbsolutePath(),
- "--index-paths=/oak:index/fooIndex",
- "--read-write=true",
- "--reindex",
- "--", // -- indicates that options have ended and rest needs to be treated as non option
- storeDir.getAbsolutePath()
- };
-
- command.execute(args);
-
- RepositoryFixture fixture2 = new RepositoryFixture(storeDir);
- NodeStore store2 = fixture2.getNodeStore();
- PropertyState reindexCount = getNode(store2.getRoot(), "/oak:index/fooIndex").getProperty(IndexConstants.REINDEX_COUNT);
- assertEquals(2, reindexCount.getValue(Type.LONG).longValue());
- }
-
-
-
}
\ No newline at end of file
Copied: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/ReindexIT.java (from r1801449, jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexImportIT.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/ReindexIT.java?p2=jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/ReindexIT.java&p1=jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexImportIT.java&r1=1801449&r2=1801451&rev=1801451&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/IndexImportIT.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/ReindexIT.java Mon Jul 10 12:00:15 2017
@@ -56,7 +56,7 @@ import static org.junit.Assert.assertNul
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
-public class IndexImportIT extends AbstractIndexCommandTest {
+public class ReindexIT extends AbstractIndexCommandTest {
@Test
public void reindexOutOfBand() throws Exception{
@@ -103,7 +103,7 @@ public class IndexImportIT extends Abstr
createTestData(true);
fixture.getAsyncIndexUpdate("async").run();
- int fooCount = getFooCount(fixture);
+ int fooCount = getFooCount(fixture, "foo");
String checkpoint = fixture.getNodeStore().checkpoint(TimeUnit.HOURS.toMillis(24));
//Close the repository so as all changes are flushed
@@ -139,7 +139,7 @@ public class IndexImportIT extends Abstr
assertThat(explain, containsString("traverse"));
assertThat(explain, not(containsString(TEST_INDEX_PATH)));
- int foo2Count = getFooCount(fixture2);
+ int foo2Count = getFooCount(fixture2, "foo");
assertEquals(fooCount + 100, foo2Count);
assertNotNull(fixture2.getNodeStore().retrieve(checkpoint));
fixture2.close();
@@ -166,7 +166,7 @@ public class IndexImportIT extends Abstr
//Phase 4 - Validate the import
RepositoryFixture fixture4 = new RepositoryFixture(storeDir);
- int foo4Count = getFooCount(fixture4);
+ int foo4Count = getFooCount(fixture4, "foo");
//new count should be same as previous
assertEquals(foo2Count, foo4Count);
@@ -184,6 +184,50 @@ public class IndexImportIT extends Abstr
fixture4.close();
}
+ @Test
+ public void reindexInReadWriteMode() throws Exception{
+ createTestData(true);
+ fixture.getAsyncIndexUpdate("async").run();
+ addTestContent(fixture, "/testNode/c", "bar", 100);
+ indexBarPropertyAlso(fixture);
+
+ String explain = getQueryPlan(fixture, "select * from [nt:base] where [bar] is not null");
+ assertThat(explain, containsString("traverse"));
+ assertThat(explain, not(containsString(TEST_INDEX_PATH)));
+
+ explain = getQueryPlan(fixture, "select * from [nt:base] where [foo] is not null");
+ assertThat(explain, containsString(TEST_INDEX_PATH));
+
+ fixture.close();
+
+ IndexCommand command = new IndexCommand();
+
+ File outDir = temporaryFolder.newFolder();
+ File storeDir = fixture.getDir();
+ String[] args = {
+ "--index-temp-dir=" + temporaryFolder.newFolder().getAbsolutePath(),
+ "--index-out-dir=" + outDir.getAbsolutePath(),
+ "--index-paths=/oak:index/fooIndex",
+ "--reindex",
+ "--read-write",
+ "--", // -- indicates that options have ended and rest needs to be treated as non option
+ storeDir.getAbsolutePath()
+ };
+
+ command.execute(args);
+
+ RepositoryFixture fixture2 = new RepositoryFixture(storeDir);
+
+ explain = getQueryPlan(fixture2, "select * from [nt:base] where [bar] is not null");
+ assertThat(explain, containsString(TEST_INDEX_PATH));
+
+ explain = getQueryPlan(fixture2, "select * from [nt:base] where [foo] is not null");
+ assertThat(explain, containsString(TEST_INDEX_PATH));
+
+ int barCount = getFooCount(fixture2, "bar");
+ assertEquals(100, barCount);
+ }
+
private void indexBarPropertyAlso(RepositoryFixture fixture2) throws IOException, RepositoryException {
Session session = fixture2.getAdminSession();
NodeState idxState = NodeStateUtils.getNode(fixture2.getNodeStore().getRoot(), TEST_INDEX_PATH);
@@ -197,10 +241,10 @@ public class IndexImportIT extends Abstr
session.logout();
}
- private int getFooCount(RepositoryFixture fixture) throws IOException, RepositoryException {
+ private int getFooCount(RepositoryFixture fixture, String propName) throws IOException, RepositoryException {
Session session = fixture.getAdminSession();
QueryManager qm = session.getWorkspace().getQueryManager();
- String explanation = getQueryPlan(fixture, "select * from [nt:base] where [foo] is not null");
+ String explanation = getQueryPlan(fixture, "select * from [nt:base] where ["+propName+"] is not null");
assertThat(explanation, containsString("/oak:index/fooIndex"));
Query q = qm.createQuery("select * from [nt:base] where [foo] is not null", Query.JCR_SQL2);
Propchange: jackrabbit/oak/trunk/oak-run/src/test/java/org/apache/jackrabbit/oak/index/ReindexIT.java
------------------------------------------------------------------------------
svn:eol-style = native