You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2014/04/02 17:17:51 UTC
svn commit: r1584067 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/
oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/
Author: jukka
Date: Wed Apr 2 15:17:50 2014
New Revision: 1584067
URL: http://svn.apache.org/r1584067
Log:
OAK-631: SegmentMK: Implement garbage collection
Use memory mapping to access already written entries in TarWriter
Improve the cache handling in SegmentTracker
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Wed Apr 2 15:17:50 2014
@@ -118,6 +118,8 @@ public class Segment {
*/
private final ConcurrentMap<Integer, Template> templates = newConcurrentMap();
+ private volatile long accessed = 0;
+
public Segment(SegmentTracker tracker, SegmentId id, ByteBuffer data) {
this.tracker = checkNotNull(tracker);
this.id = checkNotNull(id);
@@ -144,6 +146,15 @@ public class Segment {
refids[0] = id;
}
+ void access() {
+ accessed++;
+ }
+
+ boolean accessed() {
+ accessed >>>= 1;
+ return accessed != 0;
+ }
+
/**
* Maps the given record offset to the respective position within the
* internal {@link #data} array. The validity of a record with the given
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentId.java Wed Apr 2 15:17:50 2014
@@ -91,6 +91,7 @@ public class SegmentId implements Compar
}
}
}
+ segment.access();
return segment;
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentTracker.java Wed Apr 2 15:17:50 2014
@@ -21,6 +21,7 @@ import static com.google.common.collect.
import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.collect.Sets.newIdentityHashSet;
+import java.nio.ByteBuffer;
import java.security.SecureRandom;
import java.util.LinkedList;
import java.util.Queue;
@@ -70,9 +71,7 @@ public class SegmentTracker {
*/
private final SegmentIdTable[] tables = new SegmentIdTable[32];
- private final LinkedList<Segment> dataSegments = newLinkedList();
-
- private final LinkedList<Segment> bulkSegments = newLinkedList();
+ private final LinkedList<Segment> segments = newLinkedList();
private long currentSize = 0;
@@ -105,21 +104,21 @@ public class SegmentTracker {
}
void setSegment(SegmentId id, Segment segment) {
+ // done before synchronization to allow concurrent segment access
+ // while we update the cache below
id.setSegment(segment);
- LinkedList<Segment> segments = dataSegments;
- if (id.isBulkSegmentId()) {
- segments = bulkSegments;
- }
-
synchronized (this) {
- // TODO: use a scan-resistant cache
segments.addFirst(segment);
currentSize += segment.getCacheSize();
- while (currentSize > cacheSize / 2 && segments.size() > 1) {
- Segment remove = segments.removeLast();
- remove.getSegmentId().setSegment(null);
- currentSize -= remove.getCacheSize();
+ while (currentSize > cacheSize && segments.size() > 1) {
+ Segment last = segments.removeLast();
+ if (last.accessed()) {
+ segments.addFirst(last);
+ } else {
+ last.getSegmentId().setSegment(null);
+ currentSize -= last.getCacheSize();
+ }
}
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentWriter.java Wed Apr 2 15:17:50 2014
@@ -205,18 +205,19 @@ public class SegmentWriter {
buffer[pos++] = (byte) (offset >> Segment.RECORD_ALIGN_BITS);
}
- store.writeSegment(
- segment.getSegmentId(),
- buffer, buffer.length - length, length);
-
- if (length < buffer.length / 2) {
- byte[] data = new byte[length];
- System.arraycopy(
- buffer, buffer.length - length, data, 0, length);
- segment = new Segment(
- tracker, segment.getSegmentId(), ByteBuffer.wrap(data));
+ SegmentId id = segment.getSegmentId();
+ store.writeSegment(id, buffer, buffer.length - length, length);
+
+ // Keep this segment in memory as it's likely to be accessed soon
+ ByteBuffer data;
+ if (buffer.length - length > 4096) {
+ data = ByteBuffer.allocate(length);
+ data.put(buffer, buffer.length - length, length);
+ data.rewind();
+ } else {
+ data = ByteBuffer.wrap(buffer);
}
- tracker.setSegment(segment.getSegmentId(), segment);
+ tracker.setSegment(id, new Segment(tracker, id, data));
buffer = createNewBuffer();
roots.clear();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 2 15:17:50 2014
@@ -433,9 +433,13 @@ public class FileStore implements Segmen
}
synchronized (this) {
- ByteBuffer buffer = writer.readEntry(msb, lsb);
- if (buffer != null) {
- return new Segment(tracker, id, buffer);
+ try {
+ ByteBuffer buffer = writer.readEntry(msb, lsb);
+ if (buffer != null) {
+ return new Segment(tracker, id, buffer);
+ }
+ } catch (IOException e) {
+ log.warn("Failed to read from tar file " + writer, e);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntry.java Wed Apr 2 15:17:50 2014
@@ -81,4 +81,4 @@ class TarEntry {
return size;
}
-}
\ No newline at end of file
+}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Wed Apr 2 15:17:50 2014
@@ -28,6 +28,8 @@ import java.io.FileDescriptor;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel.MapMode;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
@@ -91,6 +93,8 @@ class TarWriter {
*/
private final Map<UUID, TarEntry> index = newHashMap();
+ private MappedByteBuffer buffer = null;
+
TarWriter(File file) {
this.file = file;
}
@@ -104,24 +108,20 @@ class TarWriter {
return index.containsKey(new UUID(msb, lsb));
}
- synchronized ByteBuffer readEntry(long msb, long lsb) {
+ synchronized ByteBuffer readEntry(long msb, long lsb) throws IOException {
checkState(!closed);
TarEntry entry = index.get(new UUID(msb, lsb));
if (entry != null) {
- checkState(access != null); // implied by entry != null
- try {
- try {
- byte[] data = new byte[entry.size()];
- access.seek(entry.offset());
- access.readFully(data);
- return ByteBuffer.wrap(data);
- } finally {
- access.seek(access.length());
- }
- } catch (IOException e) {
- throw new RuntimeException(
- "Unable to read from tar file " + file, e);
+ if (buffer == null
+ || buffer.remaining() < entry.offset() + entry.size()) {
+ checkState(access != null); // implied by entry != null
+ buffer = access.getChannel().map(
+ MapMode.READ_ONLY, 0, access.getFilePointer());
}
+ ByteBuffer data = buffer.asReadOnlyBuffer();
+ data.position(entry.offset());
+ data.limit(data.position() + entry.size());
+ return data.slice();
} else {
return null;
}
@@ -249,6 +249,7 @@ class TarWriter {
access.write(ZERO_BYTES);
access.write(ZERO_BYTES);
access.close();
+ buffer = null;
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java?rev=1584067&r1=1584066&r2=1584067&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/benchmark/SmallFileReadTest.java Wed Apr 2 15:17:50 2014
@@ -29,7 +29,7 @@ public class SmallFileReadTest extends A
private static final int FILE_COUNT = 1000;
- private static final int FILE_SIZE = 10;
+ private static final int FILE_SIZE = Integer.getInteger("file.size", 10);
private Session session;