You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ju...@apache.org on 2014/04/02 22:14:51 UTC
svn commit: r1584152 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file:
FileStore.java TarReader.java TarWriter.java
Author: jukka
Date: Wed Apr 2 20:14:50 2014
New Revision: 1584152
URL: http://svn.apache.org/r1584152
Log:
OAK-631: SegmentMK: Implement garbage collection
Activate the FileStore.gc() method
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1584152&r1=1584151&r2=1584152&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 2 20:14:50 2014
@@ -23,6 +23,7 @@ import static com.google.common.collect.
import static com.google.common.collect.Lists.newLinkedList;
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Maps.newTreeMap;
+import static com.google.common.collect.Sets.newHashSet;
import static java.lang.String.format;
import static java.util.Collections.singletonMap;
import static java.util.concurrent.TimeUnit.NANOSECONDS;
@@ -34,6 +35,7 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.util.Arrays;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -41,6 +43,7 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.UUID;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -113,6 +116,16 @@ public class FileStore implements Segmen
private final Thread flushThread;
/**
+ * Flag to request revision cleanup during the next flush.
+ */
+ private final AtomicBoolean cleanupNeeded = new AtomicBoolean(false);
+
+ /**
+ * List of old tar file generations that are waiting to be removed.
+ */
+ private final LinkedList<File> toBeRemoved = newLinkedList();
+
+ /**
* Synchronization aid used by the background flush thread to stop itself
* as soon as the {@link #close()} method is called.
*/
@@ -310,6 +323,39 @@ public class FileStore implements Segmen
journalFile.writeBytes(after + " root\n");
journalFile.getChannel().force(false);
persistedHead.set(after);
+
+ if (cleanupNeeded.getAndSet(false)) {
+ Set<UUID> ids = newHashSet();
+ for (SegmentId id : tracker.getReferencedSegmentIds()) {
+ ids.add(new UUID(
+ id.getMostSignificantBits(),
+ id.getLeastSignificantBits()));
+ }
+
+ List<TarReader> list =
+ newArrayListWithCapacity(readers.size());
+ for (TarReader reader : readers) {
+ TarReader cleaned = reader.cleanup(ids);
+ if (cleaned == reader) {
+ list.add(reader);
+ } else {
+ if (cleaned != null) {
+ list.add(cleaned);
+ }
+ toBeRemoved.addLast(reader.close());
+ }
+ }
+ readers = list;
+ }
+ }
+
+ // remove all obsolete tar generations
+ Iterator<File> iterator = toBeRemoved.iterator();
+ while (iterator.hasNext()) {
+ File file = iterator.next();
+ if (!file.exists() || file.delete()) {
+ iterator.remove();
+ }
}
}
}
@@ -476,6 +522,9 @@ public class FileStore implements Segmen
list.addAll(readers);
readers = list;
+ // trigger revision cleanup after next flush
+ cleanupNeeded.set(true);
+
writeNumber++;
writeFile = new File(
directory,
@@ -504,7 +553,6 @@ public class FileStore implements Segmen
@Override
public void gc() {
System.gc();
- Set<SegmentId> ids = tracker.getReferencedSegmentIds();
- // TODO reclaim unreferenced segments
+ cleanupNeeded.set(true);
}
}
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1584152&r1=1584151&r2=1584152&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Wed Apr 2 20:14:50 2014
@@ -325,7 +325,10 @@ class TarReader {
size += getEntrySize(24 * count + 16);
size += 2 * BLOCK_SIZE;
- if (size >= access.length() * 3 / 4) {
+ if (count == 0) {
+ // none of the entries within this tar file are referenceable
+ return null;
+ } else if (size >= access.length() * 3 / 4) {
// the space savings are not worth it at less than 25%
return this;
}
@@ -356,8 +359,9 @@ class TarReader {
return new TarReader(newFile, access.isMemoryMapped());
}
- void close() throws IOException {
+ File close() throws IOException {
access.close();
+ return file;
}
//-----------------------------------------------------------< private >--
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1584152&r1=1584151&r2=1584152&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Wed Apr 2 20:14:50 2014
@@ -28,8 +28,6 @@ import java.io.FileDescriptor;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel.MapMode;
import java.util.Arrays;
import java.util.Map;
import java.util.Set;
@@ -93,8 +91,6 @@ class TarWriter {
*/
private final Map<UUID, TarEntry> index = newHashMap();
- private MappedByteBuffer buffer = null;
-
TarWriter(File file) {
this.file = file;
}
@@ -112,16 +108,12 @@ class TarWriter {
checkState(!closed);
TarEntry entry = index.get(new UUID(msb, lsb));
if (entry != null) {
- if (buffer == null
- || buffer.remaining() < entry.offset() + entry.size()) {
- checkState(access != null); // implied by entry != null
- buffer = access.getChannel().map(
- MapMode.READ_ONLY, 0, access.getFilePointer());
- }
- ByteBuffer data = buffer.asReadOnlyBuffer();
- data.position(entry.offset());
- data.limit(data.position() + entry.size());
- return data.slice();
+ checkState(access != null); // implied by entry != null
+ ByteBuffer data = ByteBuffer.allocate(entry.size());
+ access.seek(entry.offset());
+ access.readFully(data.array());
+ access.seek(access.length());
+ return data;
} else {
return null;
}