You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2013/11/08 01:18:53 UTC
svn commit: r1539876 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment:
Segment.java file/FileStore.java
Author: jukka
Date: Fri Nov 8 00:18:53 2013
New Revision: 1539876
URL: http://svn.apache.org/r1539876
Log:
OAK-788: File backend for the SegmentMK
Put bulk segments in separate tar files to help pack tree content more efficiently
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1539876&r1=1539875&r2=1539876&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Fri Nov 8 00:18:53 2013
@@ -70,7 +70,7 @@ public class Segment {
* at four-byte boundaries, the two bytes can address up to 256kB of
* record data.
*/
- static final int MAX_SEGMENT_SIZE = 1 << (16 + RECORD_ALIGN_BITS); // 256kB
+ public static final int MAX_SEGMENT_SIZE = 1 << (16 + RECORD_ALIGN_BITS); // 256kB
/**
* The size limit for small values. The variable length of small values
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1539876&r1=1539875&r2=1539876&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Fri Nov 8 00:18:53 2013
@@ -45,7 +45,7 @@ public class FileStore extends AbstractS
static final UUID JOURNALS_UUID = new UUID(0, 0);
- private static final String FILE_NAME_FORMAT = "data%05d.tar";
+ private static final String FILE_NAME_FORMAT = "%s%05d.tar";
private final File directory;
@@ -53,7 +53,9 @@ public class FileStore extends AbstractS
private final boolean memoryMapping;
- private final LinkedList<TarFile> files = newLinkedList();
+ private final LinkedList<TarFile> bulkFiles = newLinkedList();
+
+ private final LinkedList<TarFile> treeFiles = newLinkedList();
private final Map<String, Journal> journals = newHashMap();
@@ -66,17 +68,27 @@ public class FileStore extends AbstractS
this.memoryMapping = memoryMapping;
for (int i = 0; true; i++) {
- String name = String.format(FILE_NAME_FORMAT, i);
+ String name = String.format(FILE_NAME_FORMAT, "bulk", i);
+ File file = new File(directory, name);
+ if (file.isFile()) {
+ bulkFiles.add(new TarFile(file, maxFileSize, memoryMapping));
+ } else {
+ break;
+ }
+ }
+
+ for (int i = 0; true; i++) {
+ String name = String.format(FILE_NAME_FORMAT, "data", i);
File file = new File(directory, name);
if (file.isFile()) {
- files.add(new TarFile(file, maxFileSize, memoryMapping));
+ treeFiles.add(new TarFile(file, maxFileSize, memoryMapping));
} else {
break;
}
}
Segment segment = getWriter().getDummySegment();
- for (TarFile tar : files) {
+ for (TarFile tar : treeFiles) {
ByteBuffer buffer = tar.readEntry(JOURNALS_UUID);
if (buffer != null) {
checkState(JOURNAL_MAGIC == buffer.getLong());
@@ -105,10 +117,14 @@ public class FileStore extends AbstractS
public synchronized void close() {
try {
super.close();
- for (TarFile file : files) {
+ for (TarFile file : bulkFiles) {
file.close();
}
- files.clear();
+ bulkFiles.clear();
+ for (TarFile file : treeFiles) {
+ file.close();
+ }
+ treeFiles.clear();
System.gc(); // for any memory-mappings that are no longer used
} catch (Exception e) {
throw new RuntimeException(e);
@@ -127,7 +143,13 @@ public class FileStore extends AbstractS
@Override
protected Segment loadSegment(UUID id) throws Exception {
- for (TarFile file : files) {
+ for (TarFile file : treeFiles) {
+ ByteBuffer buffer = file.readEntry(id);
+ if (buffer != null) {
+ return new Segment(FileStore.this, id, buffer);
+ }
+ }
+ for (TarFile file : bulkFiles) {
ByteBuffer buffer = file.readEntry(id);
if (buffer != null) {
return new Segment(FileStore.this, id, buffer);
@@ -149,9 +171,15 @@ public class FileStore extends AbstractS
private void writeEntry(
UUID segmentId, byte[] buffer, int offset, int length)
throws IOException {
+ LinkedList<TarFile> files = treeFiles;
+ String base = "data";
+ if (length == Segment.MAX_SEGMENT_SIZE) {
+ files = bulkFiles;
+ base = "bulk";
+ }
if (files.isEmpty() || !files.getLast().writeEntry(
segmentId, buffer, offset, length)) {
- String name = String.format(FILE_NAME_FORMAT, files.size());
+ String name = String.format(FILE_NAME_FORMAT, base, files.size());
File file = new File(directory, name);
TarFile last = new TarFile(file, maxFileSize, memoryMapping);
checkState(last.writeEntry(segmentId, buffer, offset, length));