You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/07/03 04:57:13 UTC
svn commit: r1800587 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/
oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugin...
Author: chetanm
Date: Mon Jul 3 04:57:13 2017
New Revision: 1800587
URL: http://svn.apache.org/viewvc?rev=1800587&view=rev
Log:
OAK-6271 - Support for importing index files
Lucene index importer implementation
Added:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporter.java (with props)
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporterTest.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java?rev=1800587&r1=1800586&r2=1800587&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporter.java Mon Jul 3 04:57:13 2017
@@ -112,7 +112,7 @@ public class IndexImporter {
mergeWithConcurrentCheck(nodeStore, builder);
}
- void importIndexData() throws CommitFailedException {
+ void importIndexData() throws CommitFailedException, IOException {
NodeState root = nodeStore.getRoot();
NodeBuilder builder = root.builder();
for (IndexInfo indexInfo : asyncLaneToIndexMapping.values()) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java?rev=1800587&r1=1800586&r2=1800587&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/index/importer/IndexImporterProvider.java Mon Jul 3 04:57:13 2017
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.plugins.index.importer;
import java.io.File;
+import java.io.IOException;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
@@ -34,7 +35,7 @@ public interface IndexImporterProvider {
* Import the index data from given directory into the
* NodeBuilder created for the index at given path
*/
- void importIndex(NodeState root, NodeBuilder defn, File indexDir);
+ void importIndex(NodeState root, NodeBuilder defn, File indexDir) throws IOException;
/**
* Index type for this implementation
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java?rev=1800587&r1=1800586&r2=1800587&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexDefinition.java Mon Jul 3 04:57:13 2017
@@ -130,12 +130,12 @@ public final class IndexDefinition imple
* Hidden node under index definition which is used to store the index definition
* nodestate as it was at time of reindexing
*/
- static final String INDEX_DEFINITION_NODE = ":index-definition";
+ public static final String INDEX_DEFINITION_NODE = ":index-definition";
/**
* Hidden node under index definition which is used to store meta info
*/
- static final String STATUS_NODE = ":status";
+ public static final String STATUS_NODE = ":status";
/**
* Property on status node which refers to the date when the index was lastUpdated
@@ -147,7 +147,7 @@ public final class IndexDefinition imple
/**
* Meta property which provides the unique id
*/
- static final String PROP_UID = "uid";
+ public static final String PROP_UID = "uid";
private static String TYPES_ALLOW_ALL_NAME = "all";
Added: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporter.java?rev=1800587&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporter.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporter.java Mon Jul 3 04:57:13 2017
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.plugins.index.importer.IndexImporterProvider;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.ReindexOperations;
+import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.IOContext;
+
+import static org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexConstants.TYPE_LUCENE;
+
+public class LuceneIndexImporter implements IndexImporterProvider {
+ private GarbageCollectableBlobStore blobStore;
+
+ public LuceneIndexImporter(){
+
+ }
+
+ public LuceneIndexImporter(GarbageCollectableBlobStore blobStore) {
+ this.blobStore = blobStore;
+ }
+
+ @Override
+ public void importIndex(NodeState root, NodeBuilder definitionBuilder, File indexDir) throws IOException {
+ LocalIndexDir localIndex = new LocalIndexDir(indexDir);
+
+ //TODO The indexFormatVersion would be considered latest. Need to be revisited
+ //if off line indexing uses older Lucene
+
+ definitionBuilder.getChildNode(IndexDefinition.STATUS_NODE);
+
+ ReindexOperations reindexOps = new ReindexOperations(root, definitionBuilder, localIndex.getJcrPath());
+ IndexDefinition definition = reindexOps.apply(true);
+
+ for (File dir : localIndex.dir.listFiles(File::isDirectory)) {
+ String jcrName = localIndex.indexMeta.getJcrNameFromFSName(dir.getName());
+ if (jcrName != null) {
+ copyDirectory(definition, definitionBuilder, jcrName, dir);
+ }
+ }
+ }
+
+ @Override
+ public String getType() {
+ return TYPE_LUCENE;
+ }
+
+ public void setBlobStore(GarbageCollectableBlobStore blobStore) {
+ this.blobStore = blobStore;
+ }
+
+ private void copyDirectory(IndexDefinition definition, NodeBuilder definitionBuilder, String jcrName, File dir)
+ throws IOException {
+ try (Closer closer = Closer.create()) {
+ Directory sourceDir = FSDirectory.open(dir);
+ closer.register(sourceDir);
+
+ //Remove any existing directory as in import case
+ //the builder can have existing hidden node structures
+ //So remove the ones which are being imported and leave
+ // //others as is
+ definitionBuilder.getChildNode(jcrName).remove();
+
+ Directory targetDir = new OakDirectory(definitionBuilder, jcrName, definition, false, blobStore);
+ closer.register(targetDir);
+
+ for (String file : sourceDir.listAll()) {
+ sourceDir.copy(targetDir, file, file, IOContext.DEFAULT);
+ }
+ }
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporter.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporterTest.java?rev=1800587&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporterTest.java (added)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporterTest.java Mon Jul 3 04:57:13 2017
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.jackrabbit.oak.InitialContent;
+import org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition;
+import org.apache.jackrabbit.oak.plugins.index.lucene.LuceneIndexEditorContext;
+import org.apache.jackrabbit.oak.plugins.index.lucene.OakDirectory;
+import org.apache.jackrabbit.oak.plugins.index.lucene.util.IndexDefinitionBuilder;
+import org.apache.jackrabbit.oak.plugins.index.lucene.writer.MultiplexersLucene;
+import org.apache.jackrabbit.oak.spi.state.EqualsDiff;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.apache.jackrabbit.oak.spi.state.NodeStateUtils;
+import org.apache.jackrabbit.oak.spi.state.ReadOnlyBuilder;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IOContext;
+import org.apache.lucene.store.IndexInput;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.PROP_UID;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.IndexDefinition.STATUS_NODE;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.createFile;
+import static org.junit.Assert.*;
+
+public class LuceneIndexImporterTest {
+ private NodeState rootState = InitialContent.INITIAL_CONTENT;
+ private NodeBuilder idx = new IndexDefinitionBuilder().build().builder();
+
+ @Rule
+ public final TemporaryFolder temporaryFolder = new TemporaryFolder(new File("target"));
+
+ @Test
+ public void exportAndImport() throws Exception{
+ NodeState baseIndexState = idx.getNodeState();
+ IndexDefinition defn = IndexDefinition.newBuilder(rootState, baseIndexState, "/oak:index/fooIndex").build();
+
+ LuceneIndexEditorContext.configureUniqueId(idx);
+
+ String dirName = ":data";
+ Directory dir = new OakDirectory(idx, dirName, defn, false);
+ createFile(dir, "foo.txt", "Test content");
+ dir.close();
+
+ String dir2Name = ":data2" + MultiplexersLucene.INDEX_DIR_SUFFIX;
+ Directory dir2 = new OakDirectory(idx, dir2Name, defn, false);
+ createFile(dir2, "foo.txt", "Test content");
+ dir2.close();
+
+ NodeBuilder builder = rootState.builder();
+ builder.child("oak:index").setChildNode("fooIndex", idx.getNodeState());
+ NodeState indexState = builder.getNodeState();
+
+ File out = temporaryFolder.newFolder();
+ LuceneIndexDumper dumper = new LuceneIndexDumper(indexState, "/oak:index/fooIndex", out);
+ dumper.dump();
+
+ LuceneIndexImporter importer = new LuceneIndexImporter();
+ NodeBuilder newBuilder = baseIndexState.builder();
+
+ //Add a file to builder to check if existing hidden nodes are removed or not
+ Directory dir3 = new OakDirectory(newBuilder, dirName, defn, false);
+ createFile(dir3, "foo2.txt", "Test content");
+ dir3.close();
+
+ importer.importIndex(rootState, newBuilder, dumper.getIndexDir());
+
+ NodeState exportedIndexState = indexState.getChildNode("oak:index").getChildNode("fooIndex");
+ NodeState importedIndexState = newBuilder.getNodeState();
+
+ assertDirectoryEquals(defn, exportedIndexState, importedIndexState, dirName);
+
+ //The uid must be different for imported directory
+ String exportedUid = exportedIndexState.getChildNode(STATUS_NODE).getString(PROP_UID);
+ String importedUid = importedIndexState.getChildNode(STATUS_NODE).getString(PROP_UID);
+ assertNotEquals(exportedUid, importedUid);
+ assertNotNull(exportedUid);
+ assertNotNull(importedUid);
+ }
+
+ private static void assertDirectoryEquals(IndexDefinition defn, NodeState expected, NodeState actual, String dirName) throws IOException {
+ OakDirectory dir1 = new OakDirectory(new ReadOnlyBuilder(expected), dirName, defn, true);
+ OakDirectory dir2 = new OakDirectory(new ReadOnlyBuilder(actual), dirName, defn, true);
+ assertDirectoryEquals(dir1, dir2);
+ dir1.close();
+ dir2.close();
+ }
+
+ private static void assertDirectoryEquals(Directory expected, Directory actual) throws IOException {
+ assertEquals(fileNameSet(expected), fileNameSet(actual));
+
+ for (String fileName : expected.listAll()) {
+ byte[] i1 = toBytes(expected.openInput(fileName, IOContext.DEFAULT));
+ byte[] i2 = toBytes(actual.openInput(fileName, IOContext.DEFAULT));
+ assertArrayEquals(i1, i2);
+ }
+ }
+
+ private static Set<List<String>> fileNameSet(Directory expected) throws IOException {
+ return ImmutableSet.of(Arrays.asList(expected.listAll()));
+ }
+
+ private static byte[] toBytes(IndexInput input) throws IOException {
+ int length = (int) input.length();
+ byte[] result = new byte[length];
+ input.readBytes(result, 0, length);
+ input.close();
+ return result;
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexImporterTest.java
------------------------------------------------------------------------------
svn:eol-style = native