You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2017/05/22 13:44:33 UTC
svn commit: r1795805 - in /jackrabbit/oak/branches/1.6: ./
oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/
Author: mduerig
Date: Mon May 22 13:44:33 2017
New Revision: 1795805
URL: http://svn.apache.org/viewvc?rev=1795805&view=rev
Log:
OAK-6208: oak-run compact should have an option to disable/enable memory mapping
merged revision 1795138
Modified:
jackrabbit/oak/branches/1.6/ (props changed)
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
Propchange: jackrabbit/oak/branches/1.6/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Mon May 22 13:44:33 2017
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1787425,1788378,1788387-1788389,1788850,1789056,1790382,1792463,1792742,1792746,1793088,1793618,1793627,1793644,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618
+/jackrabbit/oak/trunk:1781068,1781075,1781248,1781386,1781846,1781907,1782000,1782029,1782196,1782447,1782476,1782770,1782945,1782966,1782973,1782990,1783061,1783066,1783089,1783104-1783105,1783110,1783619,1783720,1783731,1783733,1783738,1783742,1783773,1783855,1783891,1784023,1784034,1784130,1784162,1784251,1784401,1784551,1784574,1784689,1785095,1785108,1785283,1785838,1785919,1785946,1787074,1787217,1787425,1788378,1788387-1788389,1788850,1789056,1790382,1792463,1792742,1792746,1793088,1793618,1793627,1793644,1795138,1795314,1795330,1795475,1795488,1795491,1795502,1795594,1795613,1795618
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java?rev=1795805&r1=1795804&r2=1795805&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java (original)
+++ jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/plugins/segment/FileStoreHelper.java Mon May 22 13:44:33 2017
@@ -36,7 +36,10 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
+import javax.annotation.Nullable;
+
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.Builder;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
import org.apache.jackrabbit.oak.plugins.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.plugins.segment.file.JournalReader;
@@ -211,6 +214,18 @@ public final class FileStoreHelper {
.withMemoryMapping(TAR_STORAGE_MEMORY_MAPPED).build();
}
+ public static FileStore openFileStore(String directory, boolean force, @Nullable Boolean mmap)
+ throws IOException, InvalidFileStoreVersionException {
+ Builder builder = FileStore.builder(checkFileStoreVersionOrFail(directory, force))
+ .withCacheSize(TAR_SEGMENT_CACHE_SIZE);
+
+ if (mmap == null && !TAR_STORAGE_MEMORY_MAPPED) {
+ return builder.withDefaultMemoryMapping().build();
+ } else {
+ return builder.withMemoryMapping((mmap != null && mmap) || TAR_STORAGE_MEMORY_MAPPED).build();
+ }
+ }
+
public static FileStore openFileStore(String directory, boolean force,
BlobStore blobStore
) throws IOException, InvalidFileStoreVersionException {
Modified: jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java?rev=1795805&r1=1795804&r2=1795805&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java (original)
+++ jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/CompactCommand.java Mon May 22 13:44:33 2017
@@ -40,9 +40,17 @@ class CompactCommand implements Command
OptionParser parser = new OptionParser();
OptionSpec<String> directoryArg = parser.nonOptions(
"Path to segment store (required)").ofType(String.class);
- OptionSpec<Void> forceFlag = parser.accepts(
- "force", "Force compaction and ignore non matching segment version");
- OptionSpec segment = parser.accepts("segment", "Use oak-segment instead of oak-segment-tar");
+ OptionSpec<Void> forceFlag = parser.accepts("force",
+ "Force compaction and ignore non matching segment version (oak-segment only)");
+ OptionSpec segment = parser.accepts("segment",
+ "Use oak-segment instead of oak-segment-tar");
+ OptionSpec<Boolean> mmapArg = parser.accepts("mmap",
+ "Use memory mapped access if true, use file access if false. " +
+ "If not specified memory mapped access is used on 64 bit systems " +
+ "and file access is used on 32 bit systems.")
+ .withOptionalArg()
+ .ofType(Boolean.class);
+
OptionSet options = parser.parse(args);
String path = directoryArg.value(options);
@@ -60,7 +68,15 @@ class CompactCommand implements Command
Set<String> afterLs = newHashSet();
Stopwatch watch = Stopwatch.createStarted();
+ Boolean mmap = mmapArg.value(options);
System.out.println("Compacting " + directory);
+ if (mmap == null) {
+ System.out.println("With default access mode");
+ } else if (mmap) {
+ System.out.println("With memory mapped access");
+ } else {
+ System.out.println("With file access");
+ }
System.out.println(" before ");
beforeLs.addAll(list(directory));
long sizeBefore = FileUtils.sizeOfDirectory(directory);
@@ -71,9 +87,9 @@ class CompactCommand implements Command
try {
if (options.has(segment)) {
- SegmentUtils.compact(directory, force);
+ SegmentUtils.compact(directory, force, mmap);
} else {
- SegmentTarUtils.compact(directory, force);
+ SegmentTarUtils.compact(directory, mmap);
}
success = true;
} catch (Throwable e) {
Modified: jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1795805&r1=1795804&r2=1795805&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original)
+++ jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Mon May 22 13:44:33 2017
@@ -28,6 +28,9 @@ import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
import com.google.common.io.Closer;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.segment.SegmentBlobReferenceRetriever;
@@ -195,10 +198,10 @@ final class SegmentTarUtils {
.run();
}
- static void compact(File directory, boolean force) {
+ static void compact(@Nonnull File directory, @Nullable Boolean mmap) {
Compact.builder()
.withPath(directory)
- .withForce(force)
+ .withMmap(mmap)
.build()
.run();
}
Modified: jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java?rev=1795805&r1=1795804&r2=1795805&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java (original)
+++ jackrabbit/oak/branches/1.6/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java Mon May 22 13:44:33 2017
@@ -57,6 +57,8 @@ import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import javax.annotation.Nullable;
+
import ch.qos.logback.classic.Level;
import ch.qos.logback.classic.Logger;
import com.google.common.collect.Maps;
@@ -166,8 +168,9 @@ class SegmentUtils {
checkConsistency(dir, journalFileName, fullTraversal, debugLevel, binLen);
}
- static void compact(File directory, boolean force) throws IOException, InvalidFileStoreVersionException {
- FileStore store = openFileStore(directory.getAbsolutePath(), force);
+ static void compact(File directory, boolean force, @Nullable Boolean mmap)
+ throws IOException, InvalidFileStoreVersionException {
+ FileStore store = openFileStore(directory.getAbsolutePath(), force, mmap);
try {
boolean persistCM = Boolean.getBoolean("tar.PersistCompactionMap");
CompactionStrategy compactionStrategy = new CompactionStrategy(
Modified: jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java?rev=1795805&r1=1795804&r2=1795805&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java (original)
+++ jackrabbit/oak/branches/1.6/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java Mon May 22 13:44:33 2017
@@ -25,7 +25,11 @@ import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
+import javax.annotation.CheckForNull;
+import javax.annotation.Nullable;
+
import org.apache.jackrabbit.oak.segment.file.FileStore;
+import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
@@ -50,7 +54,8 @@ public class Compact implements Runnable
private File path;
- private boolean force;
+ @CheckForNull
+ private Boolean mmap;
private Builder() {
// Prevent external instantiation.
@@ -68,17 +73,14 @@ public class Compact implements Runnable
}
/**
- * Set whether or not to force compact concurrent commits on top of
- * already compacted commits after the maximum number of retries has
- * been reached. Force committing tries to exclusively write lock the
- * node store.
- *
- * @param force {@code true} to force an exclusive commit of the
- * compacted state, {@code false} otherwise.
+ * Whether to use memory mapped access or file access.
+ * @param mmap {@code true} for memory mapped access, {@code false} for file access
+ * {@code null} to determine the access mode from the system architecture:
+ * memory mapped on 64 bit systems, file access on 32 bit systems.
* @return this builder.
*/
- public Builder withForce(boolean force) {
- this.force = force;
+ public Builder withMmap(@Nullable Boolean mmap) {
+ this.mmap = mmap;
return this;
}
@@ -96,8 +98,12 @@ public class Compact implements Runnable
private final File path;
+ @CheckForNull
+ private final Boolean mmap;
+
private Compact(Builder builder) {
this.path = builder.path;
+ this.mmap = builder.mmap;
}
@Override
@@ -133,7 +139,12 @@ public class Compact implements Runnable
}
private FileStore newFileStore() throws IOException, InvalidFileStoreVersionException {
- return fileStoreBuilder(path.getAbsoluteFile()).withGCOptions(defaultGCOptions().setOffline()).build();
+ FileStoreBuilder fileStoreBuilder = fileStoreBuilder(path.getAbsoluteFile())
+ .withGCOptions(defaultGCOptions().setOffline());
+
+ return mmap == null
+ ? fileStoreBuilder.build()
+ : fileStoreBuilder.withMemoryMapping(mmap).build();
}
}