You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2017/03/30 08:41:57 UTC
svn commit: r1789457 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/file/
test/java/org/apache/jackrabbit/oak/segment/file/
Author: mduerig
Date: Thu Mar 30 08:41:56 2017
New Revision: 1789457
URL: http://svn.apache.org/viewvc?rev=1789457&view=rev
Log:
OAK-6005: Add record id of the compacted root to the GC journal
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1789457&r1=1789456&r2=1789457&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Thu Mar 30 08:41:56 2017
@@ -80,7 +80,6 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentId;
-import org.apache.jackrabbit.oak.segment.SegmentIdTable;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener;
@@ -386,7 +385,7 @@ public class FileStore extends AbstractF
* @return {@code true} on success, {@code false} otherwise.
*/
public boolean compact() {
- return garbageCollector.compact() > 0;
+ return garbageCollector.compact().isSuccess();
}
/**
@@ -397,7 +396,9 @@ public class FileStore extends AbstractF
* skipping the reclaimed segments.
*/
public void cleanup() throws IOException {
- garbageCollector.cleanup();
+ CompactionResult compactionResult = CompactionResult.skipped(
+ getGcGeneration(), garbageCollector.gcOptions);
+ fileReaper.add(garbageCollector.cleanup(compactionResult));
}
/**
@@ -741,14 +742,13 @@ public class FileStore extends AbstractF
if (sufficientEstimatedGain) {
if (!gcOptions.isPaused()) {
- int gen = compact();
- if (gen > 0) {
- fileReaper.add(cleanupOldGenerations(gen));
+ CompactionResult compactionResult = compact();
+ if (compactionResult.isSuccess()) {
lastSuccessfullGC = System.currentTimeMillis();
- } else if (gen < 0) {
+ } else {
gcListener.info("TarMK GC #{}: cleaning up after failed compaction", GC_COUNT);
- fileReaper.add(cleanupGeneration(-gen));
}
+ fileReaper.add(cleanup(compactionResult));
} else {
gcListener.skipped("TarMK GC #{}: compaction paused", GC_COUNT);
}
@@ -771,17 +771,20 @@ public class FileStore extends AbstractF
stats.getApproximateSize());
}
- private int compactionAborted(int generation) {
+ @Nonnull
+ private CompactionResult compactionAborted(int generation) {
gcListener.compactionFailed(generation);
- return -generation;
+ return CompactionResult.aborted(getGcGeneration(), generation);
}
- private int compactionSucceeded(int generation) {
+ @Nonnull
+ private CompactionResult compactionSucceeded(int generation, @Nonnull RecordId compactedRootId) {
gcListener.compactionSucceeded(generation);
- return generation;
+ return CompactionResult.succeeded(generation, gcOptions, compactedRootId);
}
- synchronized int compact() {
+ @Nonnull
+ synchronized CompactionResult compact() {
final int newGeneration = getGcGeneration() + 1;
try {
Stopwatch watch = Stopwatch.createStarted();
@@ -873,7 +876,7 @@ public class FileStore extends AbstractF
writer.flush();
gcListener.info("TarMK GC #{}: compaction succeeded in {} ({} ms), after {} cycles",
GC_COUNT, watch, watch.elapsed(MILLISECONDS), cycles);
- return compactionSucceeded(newGeneration);
+ return compactionSucceeded(newGeneration, after.getRecordId());
} else {
gcListener.info("TarMK GC #{}: compaction failed after {} ({} ms), and {} cycles",
GC_COUNT, watch, watch.elapsed(MILLISECONDS), cycles);
@@ -962,43 +965,13 @@ public class FileStore extends AbstractF
: null;
}
- synchronized void cleanup() throws IOException {
- fileReaper.add(cleanupOldGenerations(getGcGeneration()));
- }
-
/**
- * Cleanup segments that are from an old generation. That segments whose generation
- * is {@code gcGeneration - SegmentGCOptions.getRetainedGenerations()} or older.
- * @param gcGeneration
+ * Cleanup segments whose generation matches the {@link CompactionResult#reclaimer()} predicate.
* @return list of files to be removed
* @throws IOException
*/
- private List<File> cleanupOldGenerations(int gcGeneration) throws IOException {
- final int reclaimGeneration = gcGeneration - gcOptions.getRetainedGenerations();
-
- Predicate<Integer> reclaimPredicate = new Predicate<Integer>() {
- @Override
- public boolean apply(Integer generation) {
- return generation <= reclaimGeneration;
- }
- };
- return cleanup(reclaimPredicate,
- "gc-count=" + GC_COUNT +
- ",gc-status=success" +
- ",store-generation=" + gcGeneration +
- ",reclaim-predicate=(generation<=" + reclaimGeneration + ")");
- }
-
- /**
- * Cleanup segments whose generation matches the {@code reclaimGeneration} predicate.
- * @param reclaimGeneration
- * @param gcInfo gc information to be passed to {@link SegmentIdTable#clearSegmentIdTables(Set, String)}
- * @return list of files to be removed
- * @throws IOException
- */
- private List<File> cleanup(
- @Nonnull Predicate<Integer> reclaimGeneration,
- @Nonnull String gcInfo)
+ @Nonnull
+ private List<File> cleanup(@Nonnull CompactionResult compactionResult)
throws IOException {
Stopwatch watch = Stopwatch.createStarted();
Set<UUID> bulkRefs = newHashSet();
@@ -1032,7 +1005,7 @@ public class FileStore extends AbstractF
Set<UUID> reclaim = newHashSet();
for (TarReader reader : cleaned.keySet()) {
- reader.mark(bulkRefs, reclaim, reclaimGeneration);
+ reader.mark(bulkRefs, reclaim, compactionResult.reclaimer());
log.info("{}: size of bulk references/reclaim set {}/{}",
reader, bulkRefs.size(), reclaim.size());
if (shutdown) {
@@ -1077,7 +1050,7 @@ public class FileStore extends AbstractF
} finally {
fileStoreLock.writeLock().unlock();
}
- tracker.clearSegmentIdTables(reclaimed, gcInfo);
+ tracker.clearSegmentIdTables(reclaimed, compactionResult.gcInfo());
// Close old readers *after* setting readers to the new readers to avoid accessing
// a closed reader from readSegment()
@@ -1092,7 +1065,9 @@ public class FileStore extends AbstractF
long finalSize = size();
long reclaimedSize = initialSize - afterCleanupSize;
stats.reclaimed(reclaimedSize);
- gcJournal.persist(reclaimedSize, finalSize, getGcGeneration(), compactionMonitor.getCompactedNodes());
+ gcJournal.persist(reclaimedSize, finalSize, getGcGeneration(),
+ compactionMonitor.getCompactedNodes(),
+ compactionResult.getCompactedRootId().toString10());
gcListener.cleaned(reclaimedSize, finalSize);
gcListener.info("TarMK GC #{}: cleanup completed in {} ({} ms). Post cleanup size is {} ({} bytes)" +
" and space reclaimed {} ({} bytes).",
@@ -1124,26 +1099,6 @@ public class FileStore extends AbstractF
}
/**
- * Cleanup segments of the given generation {@code gcGeneration}.
- * @param gcGeneration
- * @return list of files to be removed
- * @throws IOException
- */
- private List<File> cleanupGeneration(final int gcGeneration) throws IOException {
- Predicate<Integer> cleanupPredicate = new Predicate<Integer>() {
- @Override
- public boolean apply(Integer generation) {
- return generation == gcGeneration;
- }
- };
- return cleanup(cleanupPredicate,
- "gc-count=" + GC_COUNT +
- ",gc-status=failed" +
- ",store-generation=" + (gcGeneration - 1) +
- ",reclaim-predicate=(generation==" + gcGeneration + ")");
- }
-
- /**
* Finds all external blob references that are currently accessible
* in this repository and adds them to the given collector. Useful
* for collecting garbage in an external data store.
@@ -1222,4 +1177,147 @@ public class FileStore extends AbstractF
}
}
+ /**
+ * Instances of this class represent the result from a compaction.
+ * Either {@link #succeeded(int, SegmentGCOptions, RecordId) succeeded},
+ * {@link #aborted(int, int) aborted} or {@link #skipped(int, SegmentGCOptions) skipped}.
+ */
+ private abstract static class CompactionResult {
+ private final int currentGeneration;
+
+ protected CompactionResult(int currentGeneration) {
+ this.currentGeneration = currentGeneration;
+ }
+
+ /**
+ * Result of a succeeded compaction.
+ * @param newGeneration the generation successfully created by compaction
+ * @param gcOptions the current GC options used by compaction
+ * @param compactedRootId the record id of the root created by compaction
+ */
+ static CompactionResult succeeded(
+ final int newGeneration,
+ @Nonnull final SegmentGCOptions gcOptions,
+ @Nonnull final RecordId compactedRootId) {
+ return new CompactionResult(newGeneration) {
+ int oldGeneration = newGeneration - gcOptions.getRetainedGenerations();
+
+ @Override
+ Predicate<Integer> reclaimer() {
+ return CompactionResult.newOldReclaimer(oldGeneration);
+ }
+
+ @Override
+ boolean isSuccess() {
+ return true;
+ }
+
+ @Override
+ RecordId getCompactedRootId() {
+ return compactedRootId;
+ }
+ };
+ }
+
+ /**
+ * Result of an aborted compaction.
+ * @param currentGeneration the current generation of the store
+ * @param failedGeneration the generation that compaction attempted to create
+ */
+ static CompactionResult aborted(
+ int currentGeneration,
+ final int failedGeneration) {
+ return new CompactionResult(currentGeneration) {
+ @Override
+ Predicate<Integer> reclaimer() {
+ return CompactionResult.newFailedReclaimer(failedGeneration);
+ }
+
+ @Override
+ boolean isSuccess() {
+ return false;
+ }
+ };
+ }
+
+ /**
+ * Result serving as a placeholder for a compaction that was skipped.
+ * @param currentGeneration the current generation of the store
+ * @param gcOptions the current GC options used by compaction
+ */
+ static CompactionResult skipped(
+ final int currentGeneration,
+ @Nonnull final SegmentGCOptions gcOptions) {
+ return new CompactionResult(currentGeneration) {
+ int oldGeneration = currentGeneration - gcOptions.getRetainedGenerations();
+ @Override
+ Predicate<Integer> reclaimer() {
+ return CompactionResult.newOldReclaimer(oldGeneration);
+ }
+
+ @Override
+ boolean isSuccess() {
+ return true;
+ }
+ };
+ }
+
+ /**
+ * @return a predicate determining which segments to
+ * {@link GarbageCollector#cleanup(CompactionResult) clean up} for
+ * the given compaction result.
+ */
+ abstract Predicate<Integer> reclaimer();
+
+ /**
+ * @return {@code true} for {@link #succeeded(int, SegmentGCOptions, RecordId) succeeded}
+ * and {@link #skipped(int, SegmentGCOptions) skipped}, {@code false} otherwise.
+ */
+ abstract boolean isSuccess();
+
+ /**
+ * @return the record id of the compacted root on {@link #isSuccess() success},
+ * {@link RecordId#NULL} otherwise.
+ */
+ RecordId getCompactedRootId() {
+ return RecordId.NULL;
+ }
+
+ /**
+ * @return a diagnostic message describing the outcome of this compaction.
+ */
+ String gcInfo() {
+ return "gc-count=" + GC_COUNT +
+ ",gc-status=" + (isSuccess() ? "success" : "failed") +
+ ",store-generation=" + currentGeneration +
+ ",reclaim-predicate=" + reclaimer();
+ }
+
+ private static Predicate<Integer> newFailedReclaimer(final int failedGeneration) {
+ return new Predicate<Integer>() {
+ @Override
+ public boolean apply(Integer generation) {
+ return generation == failedGeneration;
+ }
+ @Override
+ public String toString() {
+ return "(generation==" + failedGeneration + ")";
+ }
+ };
+ }
+
+ private static Predicate<Integer> newOldReclaimer(final int oldGeneration) {
+ return new Predicate<Integer>() {
+ @Override
+ public boolean apply(Integer generation) {
+ return generation <= oldGeneration;
+ }
+ @Override
+ public String toString() {
+ return "(generation<=" + oldGeneration + ")";
+ }
+ };
+ }
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java?rev=1789457&r1=1789456&r2=1789457&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java Thu Mar 30 08:41:56 2017
@@ -35,8 +35,10 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
+import org.apache.jackrabbit.oak.segment.RecordId;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -70,9 +72,10 @@ public class GCJournal {
* @param repoSize current repo size
* @param gcGeneration gc generation
* @param nodes number of compacted nodes
+ * @param root record id of the compacted root node
*/
public synchronized void persist(long reclaimedSize, long repoSize,
- int gcGeneration, long nodes) {
+ int gcGeneration, long nodes, @Nonnull String root) {
GCJournalEntry current = read();
if (current.getGcGeneration() == gcGeneration) {
// failed compaction, only update the journal if the generation
@@ -80,7 +83,7 @@ public class GCJournal {
return;
}
latest = new GCJournalEntry(repoSize, reclaimedSize,
- System.currentTimeMillis(), gcGeneration, nodes);
+ System.currentTimeMillis(), gcGeneration, nodes, checkNotNull(root));
Path path = new File(directory, GC_JOURNAL).toPath();
try {
try (BufferedWriter w = newBufferedWriter(path, UTF_8, WRITE,
@@ -134,7 +137,8 @@ public class GCJournal {
public static class GCJournalEntry {
- static final GCJournalEntry EMPTY = new GCJournalEntry(-1, -1, -1, -1, -1);
+ static final GCJournalEntry EMPTY = new GCJournalEntry(
+ -1, -1, -1, -1, -1, RecordId.NULL.toString10());
private final long repoSize;
private final long reclaimedSize;
@@ -142,39 +146,54 @@ public class GCJournal {
private final int gcGeneration;
private final long nodes;
+ @Nonnull
+ private final String root;
+
public GCJournalEntry(long repoSize, long reclaimedSize, long ts,
- int gcGeneration, long nodes) {
+ int gcGeneration, long nodes, @Nonnull String root) {
this.repoSize = repoSize;
this.reclaimedSize = reclaimedSize;
this.ts = ts;
this.gcGeneration = gcGeneration;
this.nodes = nodes;
+ this.root = root;
}
@Override
public String toString() {
- return repoSize + "," + reclaimedSize + "," + ts + "," + gcGeneration + "," + nodes;
+ return repoSize + "," + reclaimedSize + "," + ts + "," + gcGeneration + "," + nodes + "," + root;
}
static GCJournalEntry fromString(String in) {
String[] items = in.split(",");
- long repoSize = safeParse(items, 0);
- long reclaimedSize = safeParse(items, 1);
- long ts = safeParse(items, 2);
- int gcGen = (int) safeParse(items, 3);
- long nodes = safeParse(items, 4);
- return new GCJournalEntry(repoSize, reclaimedSize, ts, gcGen, nodes);
+ long repoSize = parseLong(items, 0);
+ long reclaimedSize = parseLong(items, 1);
+ long ts = parseLong(items, 2);
+ int gcGen = (int) parseLong(items, 3);
+ long nodes = parseLong(items, 4);
+ String root = parseString(items, 5);
+ if (root == null) {
+ root = RecordId.NULL.toString10();
+ }
+ return new GCJournalEntry(repoSize, reclaimedSize, ts, gcGen, nodes, root);
}
- private static long safeParse(String[] items, int index) {
+ @CheckForNull
+ private static String parseString(String[] items, int index) {
if (items.length < index - 1) {
- return -1;
+ return null;
}
- String in = items[index];
- try {
- return Long.parseLong(in);
- } catch (NumberFormatException ex) {
- LOG.warn("Unable to parse {} as long value.", in, ex);
+ return items[index];
+ }
+
+ private static long parseLong(String[] items, int index) {
+ String in = parseString(items, index);
+ if (in != null) {
+ try {
+ return Long.parseLong(in);
+ } catch (NumberFormatException ex) {
+ LOG.warn("Unable to parse {} as long value.", in, ex);
+ }
}
return -1;
}
@@ -214,11 +233,20 @@ public class GCJournal {
return nodes;
}
+ /**
+ * Returns the record id of the root created by the compactor
+ */
+ @Nonnull
+ public String getRoot() {
+ return root;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + gcGeneration;
+ result = prime * result + root.hashCode();
result = prime * result + (int) (nodes ^ (nodes >>> 32));
result = prime * result + (int) (reclaimedSize ^ (reclaimedSize >>> 32));
result = prime * result + (int) (repoSize ^ (repoSize >>> 32));
@@ -253,6 +281,9 @@ public class GCJournal {
if (ts != other.ts) {
return false;
}
+ if (!root.equals(other.root)) {
+ return false;
+ }
return true;
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java?rev=1789457&r1=1789456&r2=1789457&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java Thu Mar 30 08:41:56 2017
@@ -28,6 +28,7 @@ import java.nio.file.Files;
import java.util.Collection;
import java.util.List;
+import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.file.GCJournal.GCJournalEntry;
import org.junit.Rule;
import org.junit.Test;
@@ -44,30 +45,34 @@ public class GcJournalTest {
File directory = segmentFolder.newFolder();
GCJournal gc = new GCJournal(directory);
- gc.persist(0, 100, 1, 50);
+ gc.persist(0, 100, 1, 50, RecordId.NULL.toString10());
GCJournalEntry e0 = gc.read();
assertEquals(100, e0.getRepoSize());
assertEquals(0, e0.getReclaimedSize());
assertEquals(50, e0.getNodes());
+ assertEquals(RecordId.NULL.toString10(), e0.getRoot());
- gc.persist(0, 250, 2, 75);
+ gc.persist(0, 250, 2, 75, RecordId.NULL.toString());
GCJournalEntry e1 = gc.read();
assertEquals(250, e1.getRepoSize());
assertEquals(0, e1.getReclaimedSize());
assertEquals(75, e1.getNodes());
+ assertEquals(RecordId.NULL.toString(), e1.getRoot());
- gc.persist(50, 200, 3, 90);
+ gc.persist(50, 200, 3, 90, "foo");
GCJournalEntry e2 = gc.read();
assertEquals(200, e2.getRepoSize());
assertEquals(50, e2.getReclaimedSize());
assertEquals(90, e2.getNodes());
+ assertEquals("foo", e2.getRoot());
// same gen
- gc.persist(75, 300, 3, 125);
+ gc.persist(75, 300, 3, 125, "bar");
GCJournalEntry e3 = gc.read();
assertEquals(200, e3.getRepoSize());
assertEquals(50, e3.getReclaimedSize());
assertEquals(90, e3.getNodes());
+ assertEquals("foo", e2.getRoot());
Collection<GCJournalEntry> all = gc.readAll();
assertEquals(all.size(), 3);