You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2016/01/28 05:55:56 UTC
svn commit: r1727233 - in /jackrabbit/oak/trunk/oak-lucene: pom.xml
src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
Author: chetanm
Date: Thu Jan 28 04:55:56 2016
New Revision: 1727233
URL: http://svn.apache.org/viewvc?rev=1727233&view=rev
Log:
OAK-3907 - Sync the files to directory upon copy from remote
Modified:
jackrabbit/oak/trunk/oak-lucene/pom.xml
jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
Modified: jackrabbit/oak/trunk/oak-lucene/pom.xml
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/pom.xml?rev=1727233&r1=1727232&r2=1727233&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/pom.xml (original)
+++ jackrabbit/oak/trunk/oak-lucene/pom.xml Thu Jan 28 04:55:56 2016
@@ -319,5 +319,11 @@
<groupId>org.apache.sling</groupId>
<artifactId>org.apache.sling.testing.osgi-mock</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-all</artifactId>
+ <version>1.3</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified: jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java?rev=1727233&r1=1727232&r2=1727233&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopier.java Thu Jan 28 04:55:56 2016
@@ -27,6 +27,7 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.BlockingQueue;
@@ -55,6 +56,7 @@ import com.google.common.base.Charsets;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.hash.Hashing;
@@ -388,7 +390,7 @@ public class IndexCopier implements Copy
@Override
public void run() {
scheduledForCopyCount.decrementAndGet();
- copyFilesToLocal(reference, true);
+ copyFilesToLocal(reference, true, true);
}
});
}
@@ -397,22 +399,26 @@ public class IndexCopier implements Copy
long start = PERF_LOGGER.start();
long totalSize = 0;
int copyCount = 0;
+ List<String> copiedFileNames = Lists.newArrayList();
for (String name : remote.listAll()) {
if (REMOTE_ONLY.contains(name)) {
continue;
}
CORFileReference fileRef = new CORFileReference(name);
files.putIfAbsent(name, fileRef);
- long fileSize = copyFilesToLocal(fileRef, false);
+ long fileSize = copyFilesToLocal(fileRef, false, false);
if (fileSize > 0) {
copyCount++;
totalSize += fileSize;
+ copiedFileNames.add(name);
}
}
+
+ local.sync(copiedFileNames);
PERF_LOGGER.end(start, -1, "[{}] Copied {} files totaling {}", indexPath, copyCount, humanReadableByteCount(totalSize));
}
- private long copyFilesToLocal(CORFileReference reference, boolean logDuration) {
+ private long copyFilesToLocal(CORFileReference reference, boolean sync, boolean logDuration) {
String name = reference.name;
boolean success = false;
boolean copyAttempted = false;
@@ -432,6 +438,10 @@ public class IndexCopier implements Copy
remote.copy(local, name, name, IOContext.READ);
reference.markValid();
+ if (sync) {
+ local.sync(Collections.singleton(name));
+ }
+
doneCopy(file, start);
if (logDuration) {
PERF_LOGGER.end(perfStart, 0,
Modified: jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java?rev=1727233&r1=1727232&r2=1727233&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java (original)
+++ jackrabbit/oak/trunk/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/IndexCopierTest.java Thu Jan 28 04:55:56 2016
@@ -24,6 +24,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@@ -59,7 +60,6 @@ import org.apache.lucene.store.IOContext
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.RAMDirectory;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -69,6 +69,8 @@ import static com.google.common.collect.
import static com.google.common.util.concurrent.MoreExecutors.sameThreadExecutor;
import static org.apache.jackrabbit.oak.plugins.index.IndexConstants.REINDEX_COUNT;
import static org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent.INITIAL_CONTENT;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -116,7 +118,14 @@ public class IndexCopierTest {
@Test
public void basicTestWithPrefetch() throws Exception{
- Directory baseDir = new RAMDirectory();
+ final List<String> syncedFiles = Lists.newArrayList();
+ Directory baseDir = new RAMDirectory(){
+ @Override
+ public void sync(Collection<String> names) throws IOException {
+ syncedFiles.addAll(names);
+ super.sync(names);
+ }
+ };
IndexDefinition defn = new IndexDefinition(root, builder.getNodeState());
IndexCopier c1 = new RAMIndexCopier(baseDir, sameThreadExecutor(), getWorkDir(), true);
@@ -127,6 +136,7 @@ public class IndexCopierTest {
Directory wrapped = c1.wrapForRead("/foo", defn, remote);
assertEquals(2, wrapped.listAll().length);
+ assertThat(syncedFiles, containsInAnyOrder("t1", "t2"));
assertTrue(wrapped.fileExists("t1"));
assertTrue(wrapped.fileExists("t2"));