You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2016/01/29 15:38:53 UTC
svn commit: r1727595 - in /jackrabbit/oak/trunk: oak-run/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/
oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/ oak-...
Author: alexparvulescu
Date: Fri Jan 29 14:38:52 2016
New Revision: 1727595
URL: http://svn.apache.org/viewvc?rev=1727595&view=rev
Log:
OAK-2480 Incremental (FileStore)Backup copies the entire source instead of just the delta
Modified:
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Fri Jan 29 14:38:52 2016
@@ -38,18 +38,25 @@ See the subsections below for more detai
Backup
------
-The 'backup' mode creates a backup from an existing oak repository. To start this mode, use:
+The 'backup' mode creates a backup from an existing oak repository. The most efficient
+way to backup the TarMK repository is to use a file system copy of the repository folder.
+The current backup implementation acts like a compaction to an enternal folder, on top of
+copying the state, it will also try to compress it, so it will significantly slower than
+what one might expect from a simple copy backup. Incremental backups (backup over an existing
+backup will still need to perform a full content diff) and will attempt to compact the diff.
+All optimisation flags used for offline compaction very much apply for this case as well.
+The FileStore backup doesn't need access to the DataStore, but if one is usually configured with
+the repository, it will need the following system property set to true in order to be able to
+perform the diffing `-Doak.backup.UseFakeBlobStore=true`. To start this mode, use:
- $ java -jar oak-run-*.jar backup \
- { /path/to/oak/repository | mongodb://host:port/database } /path/to/backup
+ $ java -jar oak-run-*.jar backup /path/to/oak/repository /path/to/backup
Restore
-------
The 'restore' mode imports a backup of an existing oak repository. To start this mode, use:
- $ java -jar oak-run-*.jar restore \
- { /path/to/oak/repository | mongodb://host:port/database } /path/to/backup
+ $ java -jar oak-run-*.jar restore /path/to/oak/repository /path/to/backup
Debug
-----
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Fri Jan 29 14:38:52 2016
@@ -22,6 +22,7 @@ import static org.apache.commons.io.File
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.isValidFileStoreOrFail;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openFileStore;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openReadOnlyFileStore;
+import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.newBasicReadOnlyBlobStore;
import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeGCGraph;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeSegmentGraph;
@@ -288,10 +289,18 @@ public final class Main {
}
private static void backup(String[] args) throws IOException {
+ boolean fakeBlobStore = FileStoreBackup.USE_FAKE_BLOBSTORE;
Closer closer = Closer.create();
- String h = "backup { /path/to/oak/repository | mongodb://host:port/database } <path/to/backup>";
try {
- NodeStore store = bootstrapNodeStore(args, closer, h);
+ FileStore fs;
+ if (fakeBlobStore) {
+ fs = openReadOnlyFileStore(new File(args[0]),
+ newBasicReadOnlyBlobStore());
+ } else {
+ fs = openReadOnlyFileStore(new File(args[0]));
+ }
+ closer.register(asCloseable(fs));
+ NodeStore store = SegmentNodeStore.newSegmentNodeStore(fs).create();
FileStoreBackup.backup(store, new File(args[1]));
} catch (Throwable e) {
throw closer.rethrow(e);
@@ -301,16 +310,7 @@ public final class Main {
}
private static void restore(String[] args) throws IOException {
- Closer closer = Closer.create();
- String h = "restore { /path/to/oak/repository | mongodb://host:port/database } <path/to/backup>";
- try {
- NodeStore store = bootstrapNodeStore(args, closer, h);
- FileStoreRestore.restore(new File(args[1]), store);
- } catch (Throwable e) {
- throw closer.rethrow(e);
- } finally {
- closer.close();
- }
+ FileStoreRestore.restore(new File(args[1]), new File(args[0]));
}
//TODO react to state changes of FailoverClient (triggered via JMX), once the state model of FailoverClient is complete.
Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackup.java Fri Jan 29 14:38:52 2016
@@ -18,77 +18,53 @@
*/
package org.apache.jackrabbit.oak.plugins.backup;
-import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static com.google.common.base.Preconditions.checkArgument;
+import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
import java.io.File;
import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import com.google.common.collect.ImmutableMap;
import org.apache.jackrabbit.oak.plugins.segment.Compactor;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.tooling.BasicReadOnlyBlobStore;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Stopwatch;
+
public class FileStoreBackup {
private static final Logger log = LoggerFactory
.getLogger(FileStoreBackup.class);
- private static final long DEFAULT_LIFETIME = TimeUnit.HOURS.toMillis(1);
-
- static int MAX_FILE_SIZE = 256;
+ public static boolean USE_FAKE_BLOBSTORE = Boolean.getBoolean("oak.backup.UseFakeBlobStore");
public static void backup(NodeStore store, File destination)
throws IOException {
- long s = System.currentTimeMillis();
-
- // 1. create a new checkpoint with the current state
- String checkpoint = store.checkpoint(DEFAULT_LIFETIME, ImmutableMap.of(
- "creator", FileStoreBackup.class.getSimpleName(),
- "thread", Thread.currentThread().getName()));
- NodeState current = store.retrieve(checkpoint);
- if (current == null) {
- // unable to retrieve the checkpoint; use root state instead
- current = store.getRoot();
+ checkArgument(store instanceof SegmentNodeStore);
+ Stopwatch watch = Stopwatch.createStarted();
+ NodeState current = ((SegmentNodeStore) store).getSuperRoot();
+ FileStore.Builder builder = newFileStore(destination)
+ .withDefaultMemoryMapping();
+ if (USE_FAKE_BLOBSTORE) {
+ builder.withBlobStore(new BasicReadOnlyBlobStore());
}
-
- // 2. init filestore
- FileStore backup = new FileStore(destination, MAX_FILE_SIZE, false);
+ FileStore backup = builder.create();
try {
SegmentNodeState state = backup.getHead();
- NodeState before = null;
- String beforeCheckpoint = state.getString("checkpoint");
- if (beforeCheckpoint == null) {
- // 3.1 no stored checkpoint, so do the initial full backup
- before = EMPTY_NODE;
- } else {
- // 3.2 try to retrieve the previously backed up checkpoint
- before = store.retrieve(beforeCheckpoint);
- if (before == null) {
- // the previous checkpoint is no longer available,
- // so use the backed up state as the basis of the
- // incremental backup diff
- before = state.getChildNode("root");
- }
- }
-
Compactor compactor = new Compactor(backup.getTracker());
- SegmentNodeState after = compactor.compact(before, current);
-
- // 4. commit the backup
- SegmentNodeBuilder builder = state.builder();
- builder.setProperty("checkpoint", checkpoint);
- builder.setChildNode("root", after);
- backup.setHead(state, builder.getNodeState());
+ compactor.setDeepCheckLargeBinaries(true);
+ compactor.setContentEqualityCheck(true);
+ SegmentNodeState after = compactor.compact(state, current, state);
+ backup.setHead(state, after);
} finally {
backup.close();
}
-
- log.debug("Backup finished in {} ms.", System.currentTimeMillis() - s);
+ watch.stop();
+ log.info("Backup finished in {}.", watch);
}
}
Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupRestore.java Fri Jan 29 14:38:52 2016
@@ -21,11 +21,10 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.System.nanoTime;
-import static org.apache.jackrabbit.oak.management.ManagementOperation.Status.formatTime;
import static org.apache.jackrabbit.oak.management.ManagementOperation.done;
import static org.apache.jackrabbit.oak.management.ManagementOperation.newManagementOperation;
+import static org.apache.jackrabbit.oak.management.ManagementOperation.Status.formatTime;
import static org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup.backup;
-import static org.apache.jackrabbit.oak.plugins.backup.FileStoreRestore.restore;
import java.io.File;
import java.util.concurrent.Callable;
@@ -37,14 +36,11 @@ import javax.management.openmbean.Compos
import org.apache.jackrabbit.oak.api.jmx.FileStoreBackupRestoreMBean;
import org.apache.jackrabbit.oak.management.ManagementOperation;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* Default implementation of {@link FileStoreBackupRestoreMBean} based on a file.
*/
public class FileStoreBackupRestore implements FileStoreBackupRestoreMBean {
- private static final Logger log = LoggerFactory.getLogger(FileStoreBackupRestore.class);
public static final String BACKUP_OP_NAME = "Backup";
public static final String RESTORE_OP_NAME = "Restore";
@@ -97,9 +93,7 @@ public class FileStoreBackupRestore impl
restoreOp = newManagementOperation("Restore", new Callable<String>() {
@Override
public String call() throws Exception {
- long t0 = nanoTime();
- restore(file, store);
- return "Restore completed in " + formatTime(nanoTime() - t0);
+ return "Restore not available as an online operation.";
}
});
executor.execute(restoreOp);
Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreRestore.java Fri Jan 29 14:38:52 2016
@@ -18,22 +18,20 @@
*/
package org.apache.jackrabbit.oak.plugins.backup;
+import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
+
import java.io.File;
import java.io.IOException;
-import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.plugins.segment.Compactor;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
-import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
-import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
-import org.apache.jackrabbit.oak.spi.state.NodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Stopwatch;
+
public class FileStoreRestore {
private static final Logger log = LoggerFactory
@@ -43,44 +41,30 @@ public class FileStoreRestore {
private static final String JOURNAL_FILE_NAME = "journal.log";
- public static void restore(File source, NodeStore store)
- throws IOException, CommitFailedException {
- // 1. verify that this is an actual filestore
+ public static void restore(File source, File destination)
+ throws IOException {
if (!validFileStore(source)) {
throw new IOException("Folder " + source
+ " is not a valid FileStore directory");
}
- // 2. init filestore
- FileStore restore = new FileStore(source, MAX_FILE_SIZE, false);
+ FileStore restore = new ReadOnlyStore(source);
+ Stopwatch watch = Stopwatch.createStarted();
+
+ FileStore store = newFileStore(destination).create();
+ SegmentNodeState current = store.getHead();
try {
- SegmentNodeState state = restore.getHead();
- restore(state.getChildNode("root"), store, restore);
+ Compactor compactor = new Compactor(store.getTracker());
+ compactor.setDeepCheckLargeBinaries(true);
+ SegmentNodeState after = compactor.compact(current,
+ restore.getHead(), current);
+ store.setHead(current, after);
} finally {
restore.close();
+ store.close();
}
- }
-
- private static void restore(NodeState source, NodeStore store,
- SegmentStore restore) throws CommitFailedException, IOException {
- long s = System.currentTimeMillis();
- NodeState current = store.getRoot();
- RestoreCompactor compactor = new RestoreCompactor(restore);
- SegmentNodeBuilder builder = compactor.process(current, source, current);
- store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- log.debug("Restore finished in {} ms.", System.currentTimeMillis() - s);
- }
-
- private static class RestoreCompactor extends Compactor {
-
- public RestoreCompactor(SegmentStore store) {
- super(store.getTracker());
- }
-
- @Override
- protected SegmentNodeBuilder process(NodeState before, NodeState after, NodeState onto) throws IOException {
- return super.process(before, after, onto);
- }
+ watch.stop();
+ log.info("Restore finished in {}.", watch);
}
private static boolean validFileStore(File source) {
Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Compactor.java Fri Jan 29 14:38:52 2016
@@ -64,6 +64,8 @@ public class Compactor {
r.getSegmentId().getLeastSignificantBits(), r.getOffset() };
}
+ private final SegmentTracker tracker;
+
private final SegmentWriter writer;
private final PartialCompactionMap map;
@@ -90,6 +92,21 @@ public class Compactor {
private final boolean cloneBinaries;
/**
+ * In the case of large inlined binaries, compaction will verify if all
+ * referenced segments exist in order to determine if a full clone is
+ * necessary, or just a shallow copy of the RecordId list is enough
+ * (Used in Backup scenario)
+ */
+ private boolean deepCheckLargeBinaries;
+
+ /**
+ * Flag to use content equality verification before actually compacting the
+ * state, on the childNodeChanged diff branch
+ * (Used in Backup scenario)
+ */
+ private boolean contentEqualityCheck;
+
+ /**
* Allows the cancellation of the compaction process. If this {@code
* Supplier} returns {@code true}, this compactor will cancel compaction and
* return a partial {@code SegmentNodeState} containing the changes
@@ -102,6 +119,7 @@ public class Compactor {
}
public Compactor(SegmentTracker tracker, Supplier<Boolean> cancel) {
+ this.tracker = tracker;
this.writer = tracker.getWriter();
this.map = new InMemoryCompactionMap(tracker);
this.cloneBinaries = false;
@@ -113,6 +131,7 @@ public class Compactor {
}
public Compactor(SegmentTracker tracker, CompactionStrategy compactionStrategy, Supplier<Boolean> cancel) {
+ this.tracker = tracker;
String wid = "c-" + (tracker.getCompactionMap().getGeneration() + 1);
this.writer = tracker.createSegmentWriter(wid);
if (compactionStrategy.getPersistCompactionMap()) {
@@ -135,24 +154,6 @@ public class Compactor {
/**
* Compact the differences between a {@code before} and a {@code after}
- * on top of the {@code before} state.
- * <p>
- * Equivalent to {@code compact(before, after, before)}
- *
- * @param before the before state
- * @param after the after state
- * @return the compacted state
- */
- public SegmentNodeState compact(NodeState before, NodeState after) throws IOException {
- progress.start();
- SegmentNodeState compacted = process(before, after, before).getNodeState();
- writer.flush();
- progress.stop();
- return compacted;
- }
-
- /**
- * Compact the differences between a {@code before} and a {@code after}
* on top of an {@code onto} state.
* @param before the before state
* @param after the after state
@@ -277,6 +278,10 @@ public class Compactor {
}
}
+ if (contentEqualityCheck && before.equals(after)) {
+ return true;
+ }
+
progress.onNode();
try {
NodeBuilder child = builder.getChildNode(name);
@@ -334,7 +339,7 @@ public class Compactor {
// if the blob is inlined or external, just clone it
if (sb.isExternal() || sb.length() < Segment.MEDIUM_LIMIT) {
- SegmentBlob clone = sb.clone(writer, cloneBinaries);
+ SegmentBlob clone = sb.clone(writer, false);
map.put(id, clone.getRecordId());
return clone;
}
@@ -351,8 +356,24 @@ public class Compactor {
}
}
- // if not, clone the blob and keep track of the result
- sb = sb.clone(writer, cloneBinaries);
+ boolean clone = cloneBinaries;
+ if (deepCheckLargeBinaries) {
+ clone = clone
+ || !tracker.getStore().containsSegment(
+ id.getSegmentId());
+ if (!clone) {
+ for (SegmentId bid : SegmentBlob.getBulkSegmentIds(sb)) {
+ clone = clone
+ || !tracker.getStore().containsSegment(bid);
+ if (clone) {
+ break;
+ }
+ }
+ }
+ }
+
+ // if not, clone the large blob and keep track of the result
+ sb = sb.clone(writer, clone);
map.put(id, sb.getRecordId());
if (ids == null) {
ids = newArrayList();
@@ -458,4 +479,12 @@ public class Compactor {
}
}
+ public void setDeepCheckLargeBinaries(boolean deepCheckLargeBinaries) {
+ this.deepCheckLargeBinaries = deepCheckLargeBinaries;
+ }
+
+ public void setContentEqualityCheck(boolean contentEqualityCheck) {
+ this.contentEqualityCheck = contentEqualityCheck;
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStore.java Fri Jan 29 14:38:52 2016
@@ -210,6 +210,18 @@ public class SegmentNodeStore implements
return head.get().getChildNode(ROOT);
}
+ @Nonnull
+ public NodeState getSuperRoot() {
+ if (commitSemaphore.tryAcquire()) {
+ try {
+ refreshHead();
+ } finally {
+ commitSemaphore.release();
+ }
+ }
+ return head.get();
+ }
+
@Override
public NodeState merge(
@Nonnull NodeBuilder builder, @Nonnull CommitHook commitHook,
Modified: jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Fri Jan 29 14:38:52 2016
@@ -304,6 +304,16 @@ public class FileStore implements Segmen
}
/**
+ * Set memory mapping to the default value based on OS properties
+ * @return this instance
+ */
+ @Nonnull
+ public Builder withDefaultMemoryMapping() {
+ this.memoryMapping = MEMORY_MAPPING_DEFAULT;
+ return this;
+ }
+
+ /**
* {@link GCMonitor} for monitoring this files store's gc process.
* @param gcMonitor
* @return this instance
Modified: jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java?rev=1727595&r1=1727594&r2=1727595&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/backup/FileStoreBackupTest.java Fri Jan 29 14:38:52 2016
@@ -18,17 +18,20 @@
*/
package org.apache.jackrabbit.oak.plugins.backup;
+import static org.apache.commons.io.FileUtils.deleteQuietly;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore.newSegmentNodeStore;
+import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
+import static org.junit.Assert.assertEquals;
+
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Random;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
@@ -39,10 +42,6 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import static org.apache.commons.io.FileUtils.deleteQuietly;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
public class FileStoreBackupTest {
private File src;
@@ -64,54 +63,61 @@ public class FileStoreBackupTest {
@Test
public void testBackup() throws Exception {
- FileStore source = new FileStore(src, 8, false);
+ FileStore source = newFileStore(src).withMaxFileSize(8).create();
- NodeStore store = new SegmentNodeStore(source);
+ NodeStore store = newSegmentNodeStore(source).create();
init(store);
// initial content
FileStoreBackup.backup(store, destination);
- compare(store, destination);
+ compare(source, destination);
addTestContent(store);
FileStoreBackup.backup(store, destination);
- compare(store, destination);
+ compare(source, destination);
source.close();
}
@Test
public void testRestore() throws Exception {
- FileStore source = new FileStore(src, 8, false);
+ FileStore source = newFileStore(src).withMaxFileSize(8).create();
- NodeStore store = new SegmentNodeStore(source);
+ NodeStore store = newSegmentNodeStore(source).create();
init(store);
-
- // initial content
FileStoreBackup.backup(store, destination);
-
addTestContent(store);
+ source.close();
- FileStoreRestore.restore(destination, store);
-
- compare(store, destination);
+ FileStoreRestore.restore(destination, src);
+ source = newFileStore(src).withMaxFileSize(8).create();
+ compare(source, destination);
source.close();
}
private static void addTestContent(NodeStore store)
- throws CommitFailedException {
+ throws CommitFailedException, IOException {
NodeBuilder builder = store.getRoot().builder();
- builder.child("test-backup");
+ NodeBuilder c = builder.child("test-backup").child("binaries");
+ for (int i = 0; i < 2; i++) {
+ c.setProperty("bin" + i, createBlob(store, 64 * 1024));
+ }
builder.child("root"); // make sure we don't backup the super-root
store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
- private static void compare(NodeStore store, File destination)
+ private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ byte[] data = new byte[size];
+ new Random().nextBytes(data);
+ return nodeStore.createBlob(new ByteArrayInputStream(data));
+ }
+
+ private static void compare(FileStore store, File destination)
throws IOException {
- FileStore backup = new FileStore(destination, 8, false);
- assertEquals(store.getRoot(), new SegmentNodeStore(backup).getRoot());
+ FileStore backup = newFileStore(destination).withMaxFileSize(8).create();
+ assertEquals(store.getHead(), backup.getHead());
backup.close();
}
@@ -119,45 +125,4 @@ public class FileStoreBackupTest {
new Oak(store).with(new OpenSecurityProvider())
.with(new InitialContent()).createContentRepository();
}
-
- public void testSharedContent() throws Exception {
- FileStore source = new FileStore(src, 256, false);
-
- NodeStore store = new SegmentNodeStore(source);
-
- // ~100k
- Blob blob = store.createBlob(new ByteArrayInputStream(new byte[100000]));
-
- NodeBuilder builder = store.getRoot().builder();
- NodeBuilder c1 = builder.child("test-backup");
- c1.setProperty("blob", blob);
- NodeBuilder c2 = builder.child("test-backup2");
- c2.setProperty("blob", blob);
- store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
-
- FileStoreBackup.backup(store, destination);
- compare(store, destination);
- source.close();
-
- Map<String, Long> expected = new HashMap<String, Long>();
- for (File f : src.listFiles()) {
- if (f.getName().endsWith(".tar")) {
- expected.put(f.getName(), f.length());
- }
- }
-
- for (File f : destination.listFiles()) {
- if (!f.getName().endsWith(".tar")) {
- continue;
- }
- assertTrue(f.getName() + " is missing from the backup",
- expected.containsKey(f.getName()));
- assertTrue(
- f.getName() + " is expected to have size <= "
- + expected.get(f.getName()) + " actually is "
- + f.length(),
- f.length() <= expected.get(f.getName()));
- }
-
- }
}