You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2016/05/23 11:31:04 UTC
svn commit: r1745144 - in
/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run:
CompactCommand.java SegmentTarUtils.java SegmentUtils.java
Author: frm
Date: Mon May 23 11:31:04 2016
New Revision: 1745144
URL: http://svn.apache.org/viewvc?rev=1745144&view=rev
Log:
OAK-4334 - Add a flag to choose between segment store implementations in the "compact" command
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1745144&r1=1745143&r2=1745144&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Mon May 23 11:31:04 2016
@@ -17,12 +17,8 @@
package org.apache.jackrabbit.oak.run;
-import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openFileStore;
-
import java.io.File;
-import java.io.RandomAccessFile;
import java.util.Arrays;
-import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import com.google.common.base.Stopwatch;
@@ -31,9 +27,6 @@ import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.commons.IOUtils;
-import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
-import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
-import org.apache.jackrabbit.oak.plugins.segment.file.JournalReader;
class CompactCommand implements Command {
@@ -44,6 +37,7 @@ class CompactCommand implements Command
"Path to segment store (required)").ofType(String.class);
OptionSpec<Void> forceFlag = parser.accepts(
"force", "Force compaction and ignore non matching segment version");
+ OptionSpec segmentTar = parser.accepts("segment-tar", "Use oak-segment-tar instead of oak-segment");
OptionSet options = parser.parse(args);
String path = directoryArg.value(options);
@@ -52,73 +46,18 @@ class CompactCommand implements Command
parser.printHelpOn(System.err);
System.exit(-1);
}
- Stopwatch watch = Stopwatch.createStarted();
- FileStore store = openFileStore(path, options.has(forceFlag));
+
File directory = new File(path);
- try {
- boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap");
- System.out.println("Compacting " + directory);
- System.out.println(" before " + Arrays.toString(directory.list()));
- long sizeBefore = FileUtils.sizeOfDirectory(directory);
- System.out.println(" size "
- + IOUtils.humanReadableByteCount(sizeBefore) + " (" + sizeBefore
- + " bytes)");
-
- System.out.println(" -> compacting");
-
- CompactionStrategy compactionStrategy = new CompactionStrategy(
- false, CompactionStrategy.CLONE_BINARIES_DEFAULT,
- CompactionStrategy.CleanupType.CLEAN_ALL, 0,
- CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) {
- @Override
- public boolean compacted(Callable<Boolean> setHead)
- throws Exception {
- // oak-run is doing compaction single-threaded
- // hence no guarding needed - go straight ahead
- // and call setHead
- return setHead.call();
- }
- };
- compactionStrategy.setOfflineCompaction(true);
- compactionStrategy.setPersistCompactionMap(persistCM);
- store.setCompactionStrategy(compactionStrategy);
- store.compact();
- } finally {
- store.close();
- }
+ boolean force = options.has(forceFlag);
- System.out.println(" -> cleaning up");
- store = openFileStore(path, false);
- try {
- for (File file : store.cleanup()) {
- if (!file.exists() || file.delete()) {
- System.out.println(" -> removed old file " + file.getName());
- } else {
- System.out.println(" -> failed to remove old file " + file.getName());
- }
- }
-
- String head;
- File journal = new File(directory, "journal.log");
- JournalReader journalReader = new JournalReader(journal);
- try {
- head = journalReader.iterator().next() + " root " + System.currentTimeMillis()+"\n";
- } finally {
- journalReader.close();
- }
-
- RandomAccessFile journalFile = new RandomAccessFile(journal, "rw");
- try {
- System.out.println(" -> writing new " + journal.getName() + ": " + head);
- journalFile.setLength(0);
- journalFile.writeBytes(head);
- journalFile.getChannel().force(false);
- } finally {
- journalFile.close();
- }
- } finally {
- store.close();
+ Stopwatch watch = Stopwatch.createStarted();
+
+ if (options.has(segmentTar)) {
+ SegmentTarUtils.compact(directory, force);
+ } else {
+ SegmentUtils.compact(directory, force);
}
+
watch.stop();
System.out.println(" after "
+ Arrays.toString(directory.list()));
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1745144&r1=1745143&r2=1745144&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Mon May 23 11:31:04 2016
@@ -21,6 +21,7 @@ import static com.google.common.collect.
import static com.google.common.collect.Sets.newTreeSet;
import static com.google.common.escape.Escapers.builder;
import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
+import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.checkFileStoreVersionOrFail;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.isValidFileStoreOrFail;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.newBasicReadOnlyBlobStore;
import static org.apache.jackrabbit.oak.segment.RecordType.NODE;
@@ -34,7 +35,9 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.RandomAccessFile;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
@@ -53,11 +56,13 @@ import ch.qos.logback.classic.Logger;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
import com.google.common.io.Closer;
+import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.backup.FileStoreBackup;
import org.apache.jackrabbit.oak.backup.FileStoreRestore;
+import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.json.JsopDiff;
@@ -72,6 +77,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.SegmentTracker;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStore.ReadOnlyStore;
+import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.tooling.RevisionHistory;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
@@ -153,6 +159,56 @@ class SegmentTarUtils {
checkConsistency(dir, journalFileName, fullTraversal, debugLevel, binLen);
}
+ static void compact(File directory, boolean force) throws IOException {
+ FileStore store = openFileStore(directory.getAbsolutePath(), force);
+ try {
+ boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap");
+ System.out.println("Compacting " + directory);
+ System.out.println(" before " + Arrays.toString(directory.list()));
+ long sizeBefore = FileUtils.sizeOfDirectory(directory);
+ System.out.println(" size "
+ + IOUtils.humanReadableByteCount(sizeBefore) + " (" + sizeBefore
+ + " bytes)");
+ System.out.println(" -> compacting");
+ store.compact();
+ } finally {
+ store.close();
+ }
+
+ System.out.println(" -> cleaning up");
+ store = openFileStore(directory.getAbsolutePath(), false);
+ try {
+ for (File file : store.cleanup()) {
+ if (!file.exists() || file.delete()) {
+ System.out.println(" -> removed old file " + file.getName());
+ } else {
+ System.out.println(" -> failed to remove old file " + file.getName());
+ }
+ }
+
+ String head;
+ File journal = new File(directory, "journal.log");
+ JournalReader journalReader = new JournalReader(journal);
+ try {
+ head = journalReader.iterator().next() + " root " + System.currentTimeMillis() + "\n";
+ } finally {
+ journalReader.close();
+ }
+
+ RandomAccessFile journalFile = new RandomAccessFile(journal, "rw");
+ try {
+ System.out.println(" -> writing new " + journal.getName() + ": " + head);
+ journalFile.setLength(0);
+ journalFile.writeBytes(head);
+ journalFile.getChannel().force(false);
+ } finally {
+ journalFile.close();
+ }
+ } finally {
+ store.close();
+ }
+ }
+
private static void debugFileStore(FileStore store) {
Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
int dataCount = 0;
@@ -417,6 +473,12 @@ class SegmentTarUtils {
.buildReadOnly();
}
+ private static FileStore openFileStore(String directory, boolean force) throws IOException {
+ return FileStore.builder(checkFileStoreVersionOrFail(directory, force))
+ .withCacheSize(TAR_SEGMENT_CACHE_SIZE)
+ .withMemoryMapping(TAR_STORAGE_MEMORY_MAPPED).build();
+ }
+
private static Closeable asCloseable(final FileStore fs) {
return new Closeable() {
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java?rev=1745144&r1=1745143&r2=1745144&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java Mon May 23 11:31:04 2016
@@ -20,6 +20,7 @@ package org.apache.jackrabbit.oak.run;
import static com.google.common.collect.Sets.newHashSet;
import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.newBasicReadOnlyBlobStore;
+import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openFileStore;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openReadOnlyFileStore;
import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.writeGCGraph;
@@ -31,7 +32,9 @@ import static org.slf4j.LoggerFactory.ge
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
+import java.io.RandomAccessFile;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashSet;
@@ -40,6 +43,7 @@ import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -48,6 +52,8 @@ import ch.qos.logback.classic.Logger;
import com.google.common.collect.Maps;
import com.google.common.collect.Queues;
import com.google.common.io.Closer;
+import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.commons.IOUtils;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
import org.apache.jackrabbit.oak.explorer.NodeStoreTree;
@@ -62,7 +68,9 @@ import org.apache.jackrabbit.oak.plugins
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker;
+import org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.JournalReader;
import org.apache.jackrabbit.oak.plugins.segment.file.tooling.RevisionHistory;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
@@ -136,6 +144,75 @@ class SegmentUtils {
checkConsistency(dir, journalFileName, fullTraversal, debugLevel, binLen);
}
+ static void compact(File directory, boolean force) throws IOException {
+ FileStore store = openFileStore(directory.getAbsolutePath(), force);
+ try {
+ boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap");
+ System.out.println("Compacting " + directory);
+ System.out.println(" before " + Arrays.toString(directory.list()));
+ long sizeBefore = FileUtils.sizeOfDirectory(directory);
+ System.out.println(" size "
+ + IOUtils.humanReadableByteCount(sizeBefore) + " (" + sizeBefore
+ + " bytes)");
+
+ System.out.println(" -> compacting");
+
+ CompactionStrategy compactionStrategy = new CompactionStrategy(
+ false, CompactionStrategy.CLONE_BINARIES_DEFAULT,
+ CompactionStrategy.CleanupType.CLEAN_ALL, 0,
+ CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) {
+
+ @Override
+ public boolean compacted(Callable<Boolean> setHead)
+ throws Exception {
+ // oak-run is doing compaction single-threaded
+ // hence no guarding needed - go straight ahead
+ // and call setHead
+ return setHead.call();
+ }
+ };
+ compactionStrategy.setOfflineCompaction(true);
+ compactionStrategy.setPersistCompactionMap(persistCM);
+ store.setCompactionStrategy(compactionStrategy);
+ store.compact();
+ } finally {
+ store.close();
+ }
+
+ System.out.println(" -> cleaning up");
+ store = openFileStore(directory.getAbsolutePath(), false);
+ try {
+ for (File file : store.cleanup()) {
+ if (!file.exists() || file.delete()) {
+ System.out.println(" -> removed old file " + file.getName());
+ } else {
+ System.out.println(" -> failed to remove old file " + file.getName());
+ }
+ }
+
+ String head;
+ File journal = new File(directory, "journal.log");
+ JournalReader journalReader = new JournalReader(journal);
+ try {
+ head = journalReader.iterator().next() + " root " + System.currentTimeMillis() + "\n";
+ } finally {
+ journalReader.close();
+ }
+
+ RandomAccessFile journalFile = new RandomAccessFile(journal, "rw");
+ try {
+ System.out.println(" -> writing new " + journal.getName() + ": " + head);
+ journalFile.setLength(0);
+ journalFile.writeBytes(head);
+ journalFile.getChannel().force(false);
+ } finally {
+ journalFile.close();
+ }
+ } finally {
+ store.close();
+ }
+ }
+
private static void debugFileStore(FileStore store) {
Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
int dataCount = 0;