You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2017/07/03 04:56:34 UTC
svn commit: r1800584 - in /jackrabbit/oak/trunk/oak-lucene/src:
main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/
Author: chetanm
Date: Mon Jul 3 04:56:34 2017
New Revision: 1800584
URL: http://svn.apache.org/viewvc?rev=1800584&view=rev
Log:
OAK-6271 - Support for importing index files
Add mapping for name of folder on file system to the JCR name in repository.
As the JCR name can have chars which are not suitable for FS we strip such chars
and thus ':data' is stored as 'data' on file system.
This mapping would be recorded in index-details.txt and would be later
used while importing the content back
Modified:
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactory.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMeta.java
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumper.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactoryTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMetaTest.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumperTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/DirectoryUtils.java Mon Jul 3 04:56:34 2017
@@ -90,9 +90,8 @@ public class DirectoryUtils {
return dir;
}
- static void writeMeta(File indexDir, String indexPath) throws IOException {
+ static void writeMeta(File indexDir, IndexMeta meta) throws IOException {
File readMe = new File(indexDir, INDEX_METADATA_FILE_NAME);
- IndexMeta meta = new IndexMeta(indexPath, System.currentTimeMillis());
meta.writeTo(readMe);
}
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactory.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactory.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactory.java Mon Jul 3 04:56:34 2017
@@ -43,11 +43,17 @@ public class FSDirectoryFactory implemen
String dirName, boolean reindex) throws IOException {
File indexDir = DirectoryUtils.createIndexDir(baseDir, definition.getIndexPath());
File readMe = new File(indexDir, INDEX_METADATA_FILE_NAME);
- if (!readMe.exists()) {
- DirectoryUtils.writeMeta(indexDir, definition.getIndexPath());
- }
File subDir = DirectoryUtils.createSubDir(indexDir, dirName);
FileUtils.forceMkdir(subDir);
+
+ IndexMeta meta;
+ if (!readMe.exists()) {
+ meta = new IndexMeta(definition.getIndexPath());
+ } else {
+ meta = new IndexMeta(readMe);
+ }
+ meta.addDirectoryMapping(dirName, subDir.getName());
+ DirectoryUtils.writeMeta(indexDir, meta);
return FSDirectory.open(subDir, NoLockFactory.getNoLockFactory());
}
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMeta.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMeta.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMeta.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMeta.java Mon Jul 3 04:56:34 2017
@@ -19,39 +19,87 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
-import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
-import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
import java.util.Properties;
+import javax.annotation.CheckForNull;
+
+import com.google.common.collect.Maps;
+import org.apache.commons.io.FileUtils;
+
import static com.google.common.base.Preconditions.checkNotNull;
/**
* Represents the index metadata file content as present in index-details.txt
*/
final class IndexMeta implements Comparable<IndexMeta> {
+ public static final String DIR_PREFIX = "dir.";
final String indexPath;
final long creationTime;
final int metaFormatVersion = 1;
+ private final Map<String, String> properties;
+
+ public IndexMeta(String indexPath) {
+ this(indexPath, System.currentTimeMillis(), Collections.emptyMap());
+ }
public IndexMeta(String indexPath, long creationTime) {
+ this(indexPath, creationTime, Collections.emptyMap());
+ }
+
+ public IndexMeta(String indexPath, long creationTime, Map<String, String> properties) {
this.indexPath = indexPath;
this.creationTime = creationTime;
+ this.properties = new HashMap<>(properties);
}
public IndexMeta(File file) throws IOException {
Properties p = loadFromFile(file);
this.indexPath = checkNotNull(p.getProperty("indexPath"));
this.creationTime = Long.valueOf(checkNotNull(p.getProperty("creationTime")));
+ this.properties = new HashMap<>(Maps.fromProperties(p));
+ }
+
+ public void addDirectoryMapping(String jcrDirName, String fsDirName){
+ properties.put(keyFromFSDirName(fsDirName), jcrDirName);
+ }
+
+ @CheckForNull
+ public String getJcrNameFromFSName(String fsDirName) {
+ String nameKey = keyFromFSDirName(fsDirName);
+ for (Map.Entry<String, String> e : properties.entrySet()) {
+ if (nameKey.equals(e.getKey())){
+ return e.getValue();
+ }
+ }
+ return null;
+ }
+
+ /**
+ * Returns the file system name for the given JCR name which
+ * represents a directory on the file system
+ */
+ @CheckForNull
+ public String getFSNameFromJCRName(String jcrDirName) {
+ for (Map.Entry<String, String> e : properties.entrySet()) {
+ if (e.getKey().startsWith(DIR_PREFIX) && jcrDirName.equals(e.getValue())){
+ return e.getKey().substring(DIR_PREFIX.length());
+ }
+ }
+ return null;
}
public void writeTo(File file) throws IOException {
Properties p = new Properties();
+ p.putAll(properties);
p.setProperty("metaFormatVersion", String.valueOf(metaFormatVersion));
p.setProperty("indexPath", indexPath);
p.setProperty("creationTime", String.valueOf(creationTime));
@@ -74,10 +122,14 @@ final class IndexMeta implements Compara
return String.format("%s, %tc", indexPath, creationTime);
}
+ private static String keyFromFSDirName(String fsDirName) {
+ return DIR_PREFIX + fsDirName;
+ }
+
private static Properties loadFromFile(File file) throws IOException {
InputStream is = null;
try {
- is = new BufferedInputStream(new FileInputStream(file));
+ is = FileUtils.openInputStream(file);
Properties p = new Properties();
p.load(is);
return p;
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumper.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumper.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumper.java Mon Jul 3 04:56:34 2017
@@ -62,13 +62,16 @@ public class LuceneIndexDumper {
NodeState idx = NodeStateUtils.getNode(rootState, indexPath);
IndexDefinition defn = IndexDefinition.newBuilder(rootState, idx, indexPath).build();
indexDir = DirectoryUtils.createIndexDir(baseDir, indexPath);
- DirectoryUtils.writeMeta(indexDir, indexPath);
+ IndexMeta meta = new IndexMeta(indexPath);
+
for (String dirName : idx.getChildNodeNames()) {
if (NodeStateUtils.isHidden(dirName) &&
(isIndexDirName(dirName) || isSuggestIndexDirName(dirName))) {
- copyContent(idx, defn, indexDir, dirName, closer);
+ copyContent(idx, defn, meta, indexDir, dirName, closer);
}
}
+
+ DirectoryUtils.writeMeta(indexDir, meta);
}
}
@@ -80,8 +83,11 @@ public class LuceneIndexDumper {
return indexDir;
}
- private void copyContent(NodeState idx, IndexDefinition defn, File dir, String dirName, Closer closer) throws IOException {
+ private void copyContent(NodeState idx, IndexDefinition defn, IndexMeta meta, File dir, String dirName, Closer closer) throws IOException {
File idxDir = DirectoryUtils.createSubDir(dir, dirName);
+
+ meta.addDirectoryMapping(dirName, idxDir.getName());
+
Directory sourceDir = new OakDirectory(new ReadOnlyBuilder(idx), dirName, defn, true);
Directory targetDir = FSDirectory.open(idxDir);
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactoryTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactoryTest.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactoryTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/FSDirectoryFactoryTest.java Mon Jul 3 04:56:34 2017
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.plugins.index.lucene.directory;
import java.io.File;
+import java.io.FileFilter;
import java.util.List;
import java.util.stream.Collectors;
@@ -95,4 +96,22 @@ public class FSDirectoryFactoryTest {
assertEquals(1, temporaryFolder.getRoot().list(DirectoryFileFilter.DIRECTORY).length);
}
+ @Test
+ public void directoryMapping() throws Exception{
+ IndexDefinition defn = IndexDefinition.newBuilder(root, idx.getNodeState(), "/fooIndex").build();
+ FSDirectoryFactory factory = new FSDirectoryFactory(temporaryFolder.getRoot());
+
+ Directory dir1 = factory.newInstance(defn, idx, ":data", false);
+ dir1.close();
+ Directory dir2 = factory.newInstance(defn, idx, ":some-other-data", false);
+ dir2.close();
+
+ IndexRootDirectory idxDir = new IndexRootDirectory(temporaryFolder.getRoot());
+ LocalIndexDir indexDir = idxDir.getLocalIndexes("/fooIndex").get(0);
+
+ for (File dir : indexDir.dir.listFiles((FileFilter)DirectoryFileFilter.DIRECTORY)){
+ assertNotNull(indexDir.indexMeta.getJcrNameFromFSName(dir.getName()));
+ }
+ }
+
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMetaTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMetaTest.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMetaTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/IndexMetaTest.java Mon Jul 3 04:56:34 2017
@@ -42,4 +42,17 @@ public class IndexMetaTest {
assertEquals(m.creationTime, m2.creationTime);
}
+ @Test
+ public void mapping() throws Exception{
+ IndexMeta m = new IndexMeta("/a/b", 100);
+ File f = temporaryFolder.newFile();
+ m.addDirectoryMapping(":hidden-data", "hiddendata");
+
+ assertEquals(":hidden-data", m.getJcrNameFromFSName("hiddendata"));
+ m.writeTo(f);
+ IndexMeta m2 = new IndexMeta(f);
+ assertEquals(":hidden-data", m2.getJcrNameFromFSName("hiddendata"));
+ assertEquals("hiddendata", m2.getFSNameFromJCRName(":hidden-data"));
+ }
+
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumperTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumperTest.java?rev=1800584&r1=1800583&r2=1800584&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumperTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/LuceneIndexDumperTest.java Mon Jul 3 04:56:34 2017
@@ -34,6 +34,7 @@ import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import static org.apache.jackrabbit.oak.plugins.index.lucene.TestUtil.createFile;
+import static org.apache.jackrabbit.oak.plugins.index.lucene.directory.IndexRootDirectory.INDEX_METADATA_FILE_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -71,5 +72,9 @@ public class LuceneIndexDumperTest {
assertEquals(3, indexDir.listFiles().length); // 2 dir + 1 meta
assertEquals(dumper.getSize(), size);
+
+ IndexMeta meta = new IndexMeta(new File(indexDir, INDEX_METADATA_FILE_NAME));
+ assertNotNull(meta.getFSNameFromJCRName(":data"));
+ assertNotNull(meta.getFSNameFromJCRName(":data2"+ MultiplexersLucene.INDEX_DIR_SUFFIX));
}
}
\ No newline at end of file