You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2016/01/13 17:24:18 UTC
svn commit: r1724451 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/segment/file/
test/java/org/apache/jackrabbit/oak/plugins/segment/
Author: mduerig
Date: Wed Jan 13 16:24:17 2016
New Revision: 1724451
URL: http://svn.apache.org/viewvc?rev=1724451&view=rev
Log:
OAK-3864: Filestore cleanup removes referenced segments
Add ids of segments referenced through forward references to the initial set of referenced ids
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupIT.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1724451&r1=1724450&r2=1724451&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Jan 13 16:24:17 2016
@@ -874,6 +874,7 @@ public class FileStore implements Segmen
// Do actual cleanup outside of the lock to prevent blocking
// concurrent writers for a long time
+ includeForwardReferences(cleaned.keySet(), referencedIds);
LinkedList<File> toRemove = newLinkedList();
Set<UUID> cleanedIds = newHashSet();
for (TarReader reader : cleaned.keySet()) {
@@ -934,6 +935,28 @@ public class FileStore implements Segmen
}
/**
+ * Include the ids of all segments transitively reachable through forward references from
+ * {@code referencedIds}. See OAK-3864.
+ */
+ private void includeForwardReferences(Iterable<TarReader> readers, Set<UUID> referencedIds)
+ throws IOException {
+ Set<UUID> fRefs = newHashSet(referencedIds);
+ do {
+ // Add direct forward references
+ for (TarReader reader : readers) {
+ reader.calculateForwardReferences(fRefs);
+ if (fRefs.isEmpty()) {
+ break; // Optimisation: bail out if no references left
+ }
+ }
+ if (!fRefs.isEmpty()) {
+ gcMonitor.info("TarMK GC #{}: cleanup found forward references to {}", gcCount, fRefs);
+ }
+ // ... as long as new forward references are found.
+ } while (referencedIds.addAll(fRefs));
+ }
+
+ /**
* Returns the cancellation policy for the compaction phase. If the disk
* space was considered insufficient at least once during compaction (or if
* the space was never sufficient to begin with), compaction is considered
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1724451&r1=1724450&r2=1724451&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Wed Jan 13 16:24:17 2016
@@ -709,6 +709,33 @@ class TarReader implements Closeable {
}
/**
+ * Calculate the ids of the segments directly referenced from {@code referenceIds}
+ * through forward references.
+ *
+ * @param referencedIds The initial set of ids to start from. On return it
+ * contains the set of direct forward references.
+ *
+ * @throws IOException
+ */
+ void calculateForwardReferences(Set<UUID> referencedIds) throws IOException {
+ Map<UUID, List<UUID>> graph = getGraph();
+ TarEntry[] entries = getEntries();
+ for (int i = entries.length - 1; i >= 0; i--) {
+ TarEntry entry = entries[i];
+ UUID id = new UUID(entry.msb(), entry.lsb());
+ if (referencedIds.remove(id)) {
+ if (isDataSegmentId(entry.lsb())) {
+ // this is a referenced data segment, so follow the graph
+ List<UUID> refIds = getReferences(entry, id, graph);
+ if (refIds != null) {
+ referencedIds.addAll(refIds);
+ }
+ }
+ }
+ }
+ }
+
+ /**
* Garbage collects segments in this file. First it collects the set of
* segments that are referenced / reachable, then (if more than 25% is
* garbage) creates a new generation of the file.
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupIT.java?rev=1724451&r1=1724450&r2=1724451&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupIT.java Wed Jan 13 16:24:17 2016
@@ -390,7 +390,6 @@ public class CompactionAndCleanupIT {
* This is a regression introduced with OAK-1828.
*/
@Test
- @Ignore("OAK-3864") // FIXME OAK-3864
public void cleanupCyclicGraph() throws IOException, ExecutionException, InterruptedException {
FileStore fileStore = newFileStore(directory).create();
final SegmentWriter writer = fileStore.getTracker().getWriter();