You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2016/06/08 11:44:53 UTC
svn commit: r1747384 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/backup/
test/java/org/apache/jackrabbit/oak/backup/
Author: alexparvulescu
Date: Wed Jun 8 11:44:53 2016
New Revision: 1747384
URL: http://svn.apache.org/viewvc?rev=1747384&view=rev
Log:
OAK-4278 Fix backup and restore
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreBackup.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreRestore.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/backup/FileStoreBackupTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreBackup.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreBackup.java?rev=1747384&r1=1747383&r2=1747384&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreBackup.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreBackup.java Wed Jun 8 11:44:53 2016
@@ -25,49 +25,91 @@ import static org.apache.jackrabbit.oak.
import java.io.File;
import java.io.IOException;
-import com.google.common.base.Stopwatch;
+import org.apache.jackrabbit.oak.segment.Compactor;
+import org.apache.jackrabbit.oak.segment.SegmentBufferWriter;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentWriter;
+import org.apache.jackrabbit.oak.segment.WriterCacheManager;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.tooling.BasicReadOnlyBlobStore;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Suppliers;
+
public class FileStoreBackup {
private static final Logger log = LoggerFactory
.getLogger(FileStoreBackup.class);
- public static boolean USE_FAKE_BLOBSTORE = Boolean.getBoolean("oak.backup.UseFakeBlobStore");
+ public static boolean USE_FAKE_BLOBSTORE = Boolean
+ .getBoolean("oak.backup.UseFakeBlobStore");
public static void backup(NodeStore store, File destination)
throws IOException {
checkArgument(store instanceof SegmentNodeStore);
Stopwatch watch = Stopwatch.createStarted();
- NodeState current = ((SegmentNodeStore) store).getSuperRoot();
+ SegmentGCOptions gcOptions = SegmentGCOptions.defaultGCOptions()
+ .setOffline();
+
FileStoreBuilder builder = fileStoreBuilder(destination)
.withDefaultMemoryMapping();
if (USE_FAKE_BLOBSTORE) {
builder.withBlobStore(new BasicReadOnlyBlobStore());
}
+ builder.withGCOptions(gcOptions);
FileStore backup = builder.build();
+ SegmentNodeState current = (SegmentNodeState) ((SegmentNodeStore) store)
+ .getSuperRoot();
try {
- // FIXME OAK-4278: Fix backup and restore
-// SegmentNodeState state = backup.getRevisions().getHead();
- // Use dedicated implementation instead of compactor.
- // This is allows us to decouple and fix problems for online compaction independent
- // of backup / restore.
-// Compactor compactor = new Compactor(backup.getTracker());
-// compactor.setDeepCheckLargeBinaries(true);
-// compactor.setContentEqualityCheck(true);
-// SegmentNodeState after = compactor.compact(state, current, state);
-// backup.setHead(state, after);
+ int gen = 0;
+ if (current instanceof SegmentNodeState) {
+ gen = ((SegmentNodeState) current).getRecordId().getSegment()
+ .getGcGeneration();
+ }
+ SegmentBufferWriter bufferWriter = new SegmentBufferWriter(backup,
+ backup.getTracker(), backup.getReader(), "b", gen);
+ SegmentWriter writer = new SegmentWriter(backup,
+ backup.getReader(), backup.getBlobStore(),
+ new WriterCacheManager.Default(), bufferWriter);
+ Compactor compactor = new Compactor(backup.getReader(), writer,
+ backup.getBlobStore(), Suppliers.ofInstance(false),
+ gcOptions);
+ compactor.setContentEqualityCheck(true);
+ SegmentNodeState head = backup.getReader().readHeadState();
+ SegmentNodeState after = compactor.compact(head, current, head);
+ if (after != null) {
+ backup.getRevisions().setHead(head.getRecordId(),
+ after.getRecordId());
+ }
+ } finally {
+ backup.close();
+ backup = null;
+ }
+
+ FileStoreBuilder builder2 = fileStoreBuilder(destination)
+ .withDefaultMemoryMapping();
+ builder2.withGCOptions(gcOptions);
+ backup = builder2.build();
+ try {
+ cleanup(backup);
} finally {
backup.close();
}
watch.stop();
log.info("Backup finished in {}.", watch);
}
+
+ static boolean cleanup(FileStore f) throws IOException {
+ boolean ok = true;
+ for (File file : f.cleanup()) {
+ ok = ok && file.delete();
+ }
+ return true;
+ }
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreRestore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreRestore.java?rev=1747384&r1=1747383&r2=1747384&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreRestore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/FileStoreRestore.java Wed Jun 8 11:44:53 2016
@@ -19,17 +19,26 @@
package org.apache.jackrabbit.oak.backup;
+import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
import java.io.File;
import java.io.IOException;
-import com.google.common.base.Stopwatch;
+import org.apache.jackrabbit.oak.segment.Compactor;
+import org.apache.jackrabbit.oak.segment.SegmentBufferWriter;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentWriter;
+import org.apache.jackrabbit.oak.segment.WriterCacheManager;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Stopwatch;
+import com.google.common.base.Suppliers;
+
public class FileStoreRestore {
private static final Logger log = LoggerFactory
@@ -50,17 +59,23 @@ public class FileStoreRestore {
Stopwatch watch = Stopwatch.createStarted();
FileStore store = fileStoreBuilder(destination).build();
- // FIXME OAK-4278: Fix backup and restore
-// SegmentNodeState current = store.getRevisions().getHead();
+ SegmentNodeState current = store.getReader().readHeadState();
try {
- // Use dedicated implementation instead of compactor.
- // This is allows us to decouple and fix problems for online compaction independent
- // of backup / restore.
- // compactor.setDeepCheckLargeBinaries(true);
-// Compactor compactor = new Compactor(store.getTracker());
-// SegmentNodeState after = compactor.compact(current,
-// restore.getHead(), current);
-// store.setHead(current, after);
+ SegmentNodeState head = restore.getReader().readHeadState();
+ int gen = head.getRecordId().getSegment().getGcGeneration();
+ SegmentBufferWriter bufferWriter = new SegmentBufferWriter(store,
+ store.getTracker(), store.getReader(), "r", gen);
+ SegmentWriter writer = new SegmentWriter(store, store.getReader(),
+ store.getBlobStore(), new WriterCacheManager.Default(),
+ bufferWriter);
+ SegmentGCOptions gcOptions = defaultGCOptions().setOffline();
+ Compactor compactor = new Compactor(store.getReader(), writer,
+ store.getBlobStore(), Suppliers.ofInstance(false),
+ gcOptions);
+ compactor.setContentEqualityCheck(true);
+ SegmentNodeState after = compactor.compact(current, head, current);
+ store.getRevisions().setHead(current.getRecordId(),
+ after.getRecordId());
} finally {
restore.close();
store.close();
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/backup/FileStoreBackupTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/backup/FileStoreBackupTest.java?rev=1747384&r1=1747383&r2=1747384&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/backup/FileStoreBackupTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/backup/FileStoreBackupTest.java Wed Jun 8 11:44:53 2016
@@ -31,7 +31,9 @@ import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.plugins.nodetype.write.InitialContent;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
@@ -39,13 +41,10 @@ import org.apache.jackrabbit.oak.spi.sec
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-@Ignore
-// FIXME OAK-4278: Fix backup and restore
public class FileStoreBackupTest {
private File src;
@@ -62,36 +61,41 @@ public class FileStoreBackupTest {
@Test
public void testBackup() throws Exception {
- FileStore source = fileStoreBuilder(src).withMaxFileSize(8).build();
-
- NodeStore store = SegmentNodeStoreBuilders.builder(source).build();
- init(store);
-
- // initial content
- FileStoreBackup.backup(store, destination);
-
- compare(source, destination);
-
- addTestContent(store);
- FileStoreBackup.backup(store, destination);
- compare(source, destination);
-
- source.close();
+ FileStore source = newFileStore(src);
+ SegmentNodeStore store = SegmentNodeStoreBuilders.builder(source)
+ .build();
+ try {
+ init(store);
+ source.flush();
+ FileStoreBackup.backup(store, destination);
+ compare(source, destination);
+
+ addTestContent(store);
+ source.flush();
+ FileStoreBackup.backup(store, destination);
+ compare(source, destination);
+
+ source.compact();
+ FileStoreBackup.cleanup(source);
+ FileStoreBackup.backup(store, destination);
+ compare(source, destination);
+ } finally {
+ source.close();
+ }
}
@Test
public void testRestore() throws Exception {
- FileStore source = fileStoreBuilder(src).withMaxFileSize(8).build();
-
- NodeStore store = SegmentNodeStoreBuilders.builder(source).build();
+ FileStore source = newFileStore(src);
+ SegmentNodeStore store = SegmentNodeStoreBuilders.builder(source)
+ .build();
init(store);
+ source.flush();
FileStoreBackup.backup(store, destination);
- addTestContent(store);
source.close();
FileStoreRestore.restore(destination, src);
-
- source = fileStoreBuilder(src).withMaxFileSize(8).build();
+ source = newFileStore(src);
compare(source, destination);
source.close();
}
@@ -101,22 +105,35 @@ public class FileStoreBackupTest {
NodeBuilder builder = store.getRoot().builder();
NodeBuilder c = builder.child("test-backup").child("binaries");
for (int i = 0; i < 2; i++) {
- c.setProperty("bin" + i, createBlob(store, 64 * 1024));
+ c.setProperty("binS" + i, createBlob(store, 6 * 1024));
+ }
+
+ for (int i = 0; i < 2; i++) {
+ c.setProperty("binL" + i, createBlob(store, 64 * 1024));
}
builder.child("root"); // make sure we don't backup the super-root
store.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
- private static Blob createBlob(NodeStore nodeStore, int size) throws IOException {
+ private static Blob createBlob(NodeStore nodeStore, int size)
+ throws IOException {
byte[] data = new byte[size];
new Random().nextBytes(data);
return nodeStore.createBlob(new ByteArrayInputStream(data));
}
+ private static FileStore newFileStore(File fs) throws IOException {
+ return fileStoreBuilder(fs)
+ .withMaxFileSize(1)
+ .withGCOptions(SegmentGCOptions.defaultGCOptions().setOffline())
+ .build();
+ }
+
private static void compare(FileStore store, File destination)
throws IOException {
- FileStore backup = fileStoreBuilder(destination).withMaxFileSize(8).build();
- assertEquals(store.getRevisions().getHead(), backup.getRevisions().getHead());
+ FileStore backup = fileStoreBuilder(destination).build();
+ assertEquals(store.getReader().readHeadState(), backup.getReader()
+ .readHeadState());
backup.close();
}