You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ad...@apache.org on 2018/10/16 09:25:46 UTC
svn commit: r1843987 - in /jackrabbit/oak/trunk:
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/
oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/
oak-segment-azure/src/test/java/oak/apache/jackrabbi...
Author: adulceanu
Date: Tue Oct 16 09:25:46 2018
New Revision: 1843987
URL: http://svn.apache.org/viewvc?rev=1843987&view=rev
Log:
OAK-7828 - Log off heap access for segments when creating FileStore
Modified:
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarFileTest.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarWriterTest.java
jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitPersistenceTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreProcBackend.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/FileAccess.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentNodeStorePersistence.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/AzurePersistence.java Tue Oct 16 09:25:46 2018
@@ -49,7 +49,7 @@ public class AzurePersistence implements
}
@Override
- public SegmentArchiveManager createArchiveManager(boolean mmap, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) {
+ public SegmentArchiveManager createArchiveManager(boolean mmap, boolean offHeapAccess, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) {
return new AzureArchiveManager(segmentstoreDirectory, ioMonitor, fileStoreMonitor);
}
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/SegmentCopy.java Tue Oct 16 09:25:46 2018
@@ -247,9 +247,9 @@ public class SegmentCopy {
IOMonitor ioMonitor = new IOMonitorAdapter();
FileStoreMonitor fileStoreMonitor = new FileStoreMonitorAdapter();
- SegmentArchiveManager srcArchiveManager = srcPersistence.createArchiveManager(false, ioMonitor,
+ SegmentArchiveManager srcArchiveManager = srcPersistence.createArchiveManager(false, false, ioMonitor,
fileStoreMonitor);
- SegmentArchiveManager destArchiveManager = destPersistence.createArchiveManager(false, ioMonitor,
+ SegmentArchiveManager destArchiveManager = destPersistence.createArchiveManager(false, false, ioMonitor,
fileStoreMonitor);
copyArchives(srcArchiveManager, destArchiveManager);
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/main/java/org/apache/jackrabbit/oak/segment/azure/tool/ToolUtils.java Tue Oct 16 09:25:46 2018
@@ -108,7 +108,7 @@ public class ToolUtils {
public static SegmentArchiveManager createArchiveManager(SegmentNodeStorePersistence persistence) {
SegmentArchiveManager archiveManager = null;
try {
- archiveManager = persistence.createArchiveManager(false, new IOMonitorAdapter(),
+ archiveManager = persistence.createArchiveManager(false, false, new IOMonitorAdapter(),
new FileStoreMonitorAdapter());
} catch (IOException e) {
throw new IllegalArgumentException(
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/oak/apache/jackrabbit/oak/segment/azure/tool/SegmentCopyTestBase.java Tue Oct 16 09:25:46 2018
@@ -89,9 +89,9 @@ public abstract class SegmentCopyTestBas
IOMonitor ioMonitor = new IOMonitorAdapter();
FileStoreMonitor fileStoreMonitor = new FileStoreMonitorAdapter();
- SegmentArchiveManager srcArchiveManager = srcPersistence.createArchiveManager(false, ioMonitor,
+ SegmentArchiveManager srcArchiveManager = srcPersistence.createArchiveManager(false, false, ioMonitor,
fileStoreMonitor);
- SegmentArchiveManager destArchiveManager = destPersistence.createArchiveManager(false, ioMonitor,
+ SegmentArchiveManager destArchiveManager = destPersistence.createArchiveManager(false, false, ioMonitor,
fileStoreMonitor);
checkArchives(srcArchiveManager, destArchiveManager);
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureArchiveManagerTest.java Tue Oct 16 09:25:46 2018
@@ -61,7 +61,7 @@ public class AzureArchiveManagerTest {
@Test
public void testRecovery() throws StorageException, URISyntaxException, IOException {
- SegmentArchiveManager manager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
+ SegmentArchiveManager manager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(false, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
SegmentArchiveWriter writer = manager.create("data00000a.tar");
List<UUID> uuids = new ArrayList<>();
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarFileTest.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarFileTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarFileTest.java Tue Oct 16 09:25:46 2018
@@ -42,7 +42,7 @@ public class AzureTarFileTest extends Ta
public void setUp() throws IOException {
try {
container = azurite.getContainer("oak-test");
- archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
+ archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
} catch (StorageException | InvalidKeyException | URISyntaxException e) {
throw new IOException(e);
}
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarWriterTest.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarWriterTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/azure/AzureTarWriterTest.java Tue Oct 16 09:25:46 2018
@@ -40,7 +40,7 @@ public class AzureTarWriterTest extends
try {
monitor = new TestFileStoreMonitor();
container = azurite.getContainer("oak-test");
- archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, new IOMonitorAdapter(), monitor);
+ archiveManager = new AzurePersistence(container.getDirectoryReference("oak")).createArchiveManager(true, false, new IOMonitorAdapter(), monitor);
} catch (StorageException | InvalidKeyException | URISyntaxException e) {
throw new IOException(e);
}
Modified: jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitPersistenceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitPersistenceTest.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitPersistenceTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-azure/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitPersistenceTest.java Tue Oct 16 09:25:46 2018
@@ -134,7 +134,7 @@ public class SplitPersistenceTest {
splitFileStore.close();
splitFileStore = null;
- SegmentArchiveManager manager = splitPersistence.createArchiveManager(true, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
+ SegmentArchiveManager manager = splitPersistence.createArchiveManager(true, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
for (String archive : manager.listArchives()) {
SegmentArchiveReader reader = manager.open(archive);
BinaryReferencesIndexLoader.parseBinaryReferencesIndex(reader.getBinaryReferences());
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java Tue Oct 16 09:25:46 2018
@@ -109,6 +109,8 @@ public abstract class AbstractFileStore
final boolean memoryMapping;
+ final boolean offHeapAccess;
+
@NotNull
final SegmentCache segmentCache;
@@ -144,6 +146,7 @@ public abstract class AbstractFileStore
builder.getStatsProvider().getMeter("oak.segment.reads", StatsOptions.DEFAULT)
);
this.memoryMapping = builder.getMemoryMapping();
+ this.offHeapAccess = builder.getOffHeapAccess();
this.ioMonitor = builder.getIOMonitor();
this.segmentBufferMonitor = new SegmentBufferMonitor(builder.getStatsProvider());
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Oct 16 09:25:46 2018
@@ -194,9 +194,10 @@ public class FileStore extends AbstractF
}
});
- log.info("TarMK opened at {}, mmap={}, size={}",
+ log.info("TarMK opened at {}, mmap={}, offHeapAccess={}, size={}",
directory,
memoryMapping,
+ offHeapAccess,
newPrintableBytes(size)
);
log.debug("TAR files: {}", tarFiles);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Tue Oct 16 09:25:46 2018
@@ -22,6 +22,7 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.Sets.newHashSet;
+import static java.lang.Boolean.getBoolean;
import static org.apache.jackrabbit.oak.segment.CachingSegmentReader.DEFAULT_STRING_CACHE_MB;
import static org.apache.jackrabbit.oak.segment.CachingSegmentReader.DEFAULT_TEMPLATE_CACHE_MB;
import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB;
@@ -36,6 +37,7 @@ import java.io.IOException;
import java.util.Set;
import com.google.common.base.Predicate;
+
import org.apache.jackrabbit.oak.segment.CacheWeights.NodeCacheWeigher;
import org.apache.jackrabbit.oak.segment.CacheWeights.StringCacheWeigher;
import org.apache.jackrabbit.oak.segment.CacheWeights.TemplateCacheWeigher;
@@ -95,6 +97,8 @@ public class FileStoreBuilder {
private boolean memoryMapping = MEMORY_MAPPING_DEFAULT;
+ private boolean offHeapAccess = getBoolean("access.off.heap");
+
private SegmentNodeStorePersistence persistence;
@NotNull
@@ -133,7 +137,7 @@ public class FileStoreBuilder {
private final Set<IOMonitor> ioMonitors = newHashSet();
private boolean strictVersionCheck;
-
+
private boolean built;
/**
@@ -252,6 +256,17 @@ public class FileStoreBuilder {
}
/**
+ * Turn off heap access on or off
+ * @param offHeapAccess
+ * @return this instance
+ */
+ @NotNull
+ public FileStoreBuilder withOffHeapAccess(boolean offHeapAccess) {
+ this.offHeapAccess = offHeapAccess;
+ return this;
+ }
+
+ /**
* Set memory mapping to the default value based on OS properties
* @return this instance
*/
@@ -337,7 +352,7 @@ public class FileStoreBuilder {
this.strictVersionCheck = strictVersionCheck;
return this;
}
-
+
public FileStoreBuilder withCustomPersistence(SegmentNodeStorePersistence persistence) throws IOException {
this.persistence = persistence;
return this;
@@ -456,6 +471,10 @@ public class FileStoreBuilder {
return memoryMapping;
}
+ boolean getOffHeapAccess() {
+ return offHeapAccess;
+ }
+
@NotNull
GCListener getGcListener() {
return gcListener;
@@ -470,7 +489,7 @@ public class FileStoreBuilder {
SegmentGCOptions getGcOptions() {
return gcOptions;
}
-
+
@NotNull
SegmentNotFoundExceptionListener getSnfeListener() {
return snfeListener;
@@ -521,6 +540,7 @@ public class FileStoreBuilder {
", templateDeduplicationCacheSize=" + templateDeduplicationCacheSize +
", nodeDeduplicationCacheSize=" + nodeDeduplicationCacheSize +
", memoryMapping=" + memoryMapping +
+ ", offHeapAccess=" + offHeapAccess +
", gcOptions=" + gcOptions +
'}';
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreProcBackend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreProcBackend.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreProcBackend.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreProcBackend.java Tue Oct 16 09:25:46 2018
@@ -55,7 +55,7 @@ class FileStoreProcBackend implements Ba
FileStoreProcBackend(AbstractFileStore fileStore, SegmentNodeStorePersistence persistence) throws IOException {
this.fileStore = fileStore;
this.persistence = persistence;
- this.archiveManager = persistence.createArchiveManager(true, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
+ this.archiveManager = persistence.createArchiveManager(true, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
}
@Override
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/FileAccess.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/FileAccess.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/FileAccess.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/FileAccess.java Tue Oct 16 09:25:46 2018
@@ -19,7 +19,6 @@
package org.apache.jackrabbit.oak.segment.file.tar;
import static com.google.common.base.Preconditions.checkState;
-import static java.lang.Boolean.getBoolean;
import static java.nio.channels.FileChannel.MapMode.READ_ONLY;
import static org.apache.jackrabbit.oak.commons.IOUtils.readFully;
@@ -36,8 +35,6 @@ import java.nio.channels.FileChannel;
*/
abstract class FileAccess {
- private static final boolean OFF_HEAP = getBoolean("access.off.heap");
-
abstract boolean isMemoryMapped();
abstract int length() throws IOException;
@@ -87,14 +84,14 @@ abstract class FileAccess {
}
}
-
+
/**
* The implementation that uses random access file (reads are synchronized).
- */
+ */
static class Random extends FileAccess {
private final RandomAccessFile file;
- private final FileChannel channel;
+ protected final FileChannel channel;
Random(RandomAccessFile file) {
this.file = file;
@@ -117,11 +114,8 @@ abstract class FileAccess {
public synchronized ByteBuffer read(int position, int length)
throws IOException {
ByteBuffer entry;
- if (OFF_HEAP) {
- entry = ByteBuffer.allocateDirect(length);
- } else {
- entry = ByteBuffer.allocate(length);
- }
+ entry = ByteBuffer.allocate(length);
+
if (readFully(channel, position, entry) < length) {
throw new EOFException();
}
@@ -136,4 +130,28 @@ abstract class FileAccess {
}
+ /**
+ * The implementation that uses random access file (reads are synchronized)
+ * and off heap access.
+ */
+ static class RandomOffHeap extends Random {
+
+ RandomOffHeap(RandomAccessFile file) {
+ super(file);
+ }
+
+ @Override
+ public synchronized ByteBuffer read(int position, int length)
+ throws IOException {
+ ByteBuffer entry;
+ entry = ByteBuffer.allocateDirect(length);
+
+ if (readFully(channel, position, entry) < length) {
+ throw new EOFException();
+ }
+ entry.flip();
+ return entry;
+ }
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java Tue Oct 16 09:25:46 2018
@@ -18,11 +18,15 @@
*/
package org.apache.jackrabbit.oak.segment.file.tar;
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.ByteBuffer.wrap;
+import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BLOCK_SIZE;
+
import org.apache.commons.io.filefilter.SuffixFileFilter;
+import org.apache.jackrabbit.oak.segment.file.tar.index.Index;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor;
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
-import org.apache.jackrabbit.oak.segment.file.tar.index.Index;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveReader;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter;
import org.slf4j.Logger;
@@ -41,10 +45,6 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.CRC32;
-import static com.google.common.base.Charsets.UTF_8;
-import static java.nio.ByteBuffer.wrap;
-import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.BLOCK_SIZE;
-
public class SegmentTarManager implements SegmentArchiveManager {
/**
@@ -65,11 +65,15 @@ public class SegmentTarManager implement
private final boolean memoryMapping;
- public SegmentTarManager(File segmentstoreDir, FileStoreMonitor fileStoreMonitor, IOMonitor ioMonitor, boolean memoryMapping) {
+ private final boolean offHeapAccess;
+
+ public SegmentTarManager(File segmentstoreDir, FileStoreMonitor fileStoreMonitor, IOMonitor ioMonitor, boolean memoryMapping,
+ boolean offHeapAccess) {
this.segmentstoreDir = segmentstoreDir;
this.fileStoreMonitor = fileStoreMonitor;
this.ioMonitor = ioMonitor;
this.memoryMapping = memoryMapping;
+ this.offHeapAccess = offHeapAccess;
}
@Override
@@ -102,7 +106,13 @@ public class SegmentTarManager implement
}
}
- FileAccess random = new FileAccess.Random(access);
+ FileAccess random = null;
+ if (offHeapAccess) {
+ random = new FileAccess.RandomOffHeap(access);
+ } else {
+ random = new FileAccess.Random(access);
+ }
+
// prevent the finally block from closing the file
// as the returned TarReader will take care of that
access = null;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java Tue Oct 16 09:25:46 2018
@@ -112,6 +112,8 @@ public class TarFiles implements Closeab
private boolean memoryMapping;
+ private boolean offHeapAccess;
+
private TarRecovery tarRecovery;
private IOMonitor ioMonitor;
@@ -138,6 +140,11 @@ public class TarFiles implements Closeab
return this;
}
+ public Builder withOffHeapAccess(boolean offHeapAccess) {
+ this.offHeapAccess = offHeapAccess;
+ return this;
+ }
+
public Builder withTarRecovery(TarRecovery tarRecovery) {
this.tarRecovery = checkNotNull(tarRecovery);
return this;
@@ -210,7 +217,7 @@ public class TarFiles implements Closeab
}
private SegmentArchiveManager buildArchiveManager() throws IOException {
- return persistence.createArchiveManager(memoryMapping, ioMonitor, readOnly && fileStoreMonitor == null ? new FileStoreMonitorAdapter() : fileStoreMonitor);
+ return persistence.createArchiveManager(memoryMapping, offHeapAccess, ioMonitor, readOnly && fileStoreMonitor == null ? new FileStoreMonitorAdapter() : fileStoreMonitor);
}
}
@@ -354,6 +361,7 @@ public class TarFiles implements Closeab
writer = new TarWriter(archiveManager, writeNumber);
}
+ @Override
public void close() throws IOException {
shutdown = true;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java Tue Oct 16 09:25:46 2018
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.file.tar;
import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.segment.file.LocalGCJournalFile;
+import org.apache.jackrabbit.oak.segment.file.LocalManifestFile;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor;
import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile;
@@ -27,8 +29,6 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.spi.persistence.RepositoryLock;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
-import org.apache.jackrabbit.oak.segment.file.LocalGCJournalFile;
-import org.apache.jackrabbit.oak.segment.file.LocalManifestFile;
import java.io.File;
import java.io.IOException;
@@ -54,8 +54,9 @@ public class TarPersistence implements S
}
@Override
- public SegmentArchiveManager createArchiveManager(boolean memoryMapping, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) {
- return new SegmentTarManager(directory, fileStoreMonitor, ioMonitor, memoryMapping);
+ public SegmentArchiveManager createArchiveManager(boolean memoryMapping, boolean offHeapAccess, IOMonitor ioMonitor,
+ FileStoreMonitor fileStoreMonitor) {
+ return new SegmentTarManager(directory, fileStoreMonitor, ioMonitor, memoryMapping, offHeapAccess);
}
@Override
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentNodeStorePersistence.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentNodeStorePersistence.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentNodeStorePersistence.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/spi/persistence/SegmentNodeStorePersistence.java Tue Oct 16 09:25:46 2018
@@ -35,6 +35,7 @@ public interface SegmentNodeStorePersist
*
* @param memoryMapping whether the memory mapping should be used (if the given
* persistence supports it)
+ * @param offHeapAccess whether off heap access for segements should be used
* @param ioMonitor object used to monitor segment-related IO access. The
* implementation should call the appropriate methods when
* accessing segments.
@@ -42,7 +43,8 @@ public interface SegmentNodeStorePersist
* @return segment archive manager
* @throws IOException
*/
- SegmentArchiveManager createArchiveManager(boolean memoryMapping, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) throws IOException;
+ SegmentArchiveManager createArchiveManager(boolean memoryMapping, boolean offHeapAccess, IOMonitor ioMonitor,
+ FileStoreMonitor fileStoreMonitor) throws IOException;
/**
* Check if the segment store already contains any segments
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java Tue Oct 16 09:25:46 2018
@@ -86,7 +86,7 @@ public class SplitPersistence implements
}
private Optional<String> getLastArchive() throws IOException {
- SegmentArchiveManager manager = roPersistence.createArchiveManager(false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
+ SegmentArchiveManager manager = roPersistence.createArchiveManager(false, false, new IOMonitorAdapter(), new FileStoreMonitorAdapter());
List<String> archives = manager.listArchives();
if (archives.isEmpty()) {
return Optional.empty();
@@ -97,14 +97,14 @@ public class SplitPersistence implements
}
@Override
- public SegmentArchiveManager createArchiveManager(boolean memoryMapping, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) throws IOException {
+ public SegmentArchiveManager createArchiveManager(boolean memoryMapping, boolean offHeapAccess, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) throws IOException {
if (lastRoArchive.isPresent()) {
return new SplitSegmentArchiveManager(
- roPersistence.createArchiveManager(memoryMapping, ioMonitor, fileStoreMonitor),
- rwPersistence.createArchiveManager(memoryMapping, ioMonitor, fileStoreMonitor),
+ roPersistence.createArchiveManager(memoryMapping, offHeapAccess, ioMonitor, fileStoreMonitor),
+ rwPersistence.createArchiveManager(memoryMapping, offHeapAccess, ioMonitor, fileStoreMonitor),
lastRoArchive.get());
} else {
- return rwPersistence.createArchiveManager(memoryMapping, ioMonitor, fileStoreMonitor);
+ return rwPersistence.createArchiveManager(memoryMapping, offHeapAccess, ioMonitor, fileStoreMonitor);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java Tue Oct 16 09:25:46 2018
@@ -58,7 +58,7 @@ public class TarFileTest {
@Before
public void setUp() throws IOException {
- archiveManager = new SegmentTarManager(folder.newFolder(), new FileStoreMonitorAdapter(), new IOMonitorAdapter(), false);
+ archiveManager = new SegmentTarManager(folder.newFolder(), new FileStoreMonitorAdapter(), new IOMonitorAdapter(), false, false);
}
protected long getWriteAndReadExpectedSize() {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java?rev=1843987&r1=1843986&r2=1843987&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java Tue Oct 16 09:25:46 2018
@@ -49,7 +49,7 @@ public class TarWriterTest {
@Before
public void setUp() throws IOException {
monitor = new TestFileStoreMonitor();
- archiveManager = new SegmentTarManager(folder.newFolder(), monitor, new IOMonitorAdapter(), false);
+ archiveManager = new SegmentTarManager(folder.newFolder(), monitor, new IOMonitorAdapter(), false, false);
}
@Test