You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2017/07/27 09:05:16 UTC
svn commit: r1803145 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/
main/java/org/apache/jackrabbit/oak/segment/compaction/
main/java/org/apache/jackrabbit/oak/segment/file/
main/java/org/apache/jackrabbit/...
Author: frm
Date: Thu Jul 27 09:05:15 2017
New Revision: 1803145
URL: http://svn.apache.org/viewvc?rev=1803145&view=rev
Log:
OAK-3349 - Core integration between full and tail compaction
Contribution by Michael Dürig.
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/GCGeneration.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyClientSync.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CacheWeightEstimator.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java Thu Jul 27 09:05:15 2017
@@ -271,7 +271,8 @@ public class DefaultSegmentWriter implem
SegmentWriteOperation with(@Nonnull SegmentBufferWriter writer) {
checkState(this.writer == null);
this.writer = writer;
- int generation = writer.getGeneration().getGeneration();
+ // FIXME OAK-3349 Also take the tail part of the gc generation into account for allocating cache generations. Cache generations need to be a monotonically increasing, ordered sequence consisting of the full and tail part of the gc generation. See also org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.EvictingWriteCacheManager.evictOldGeneration
+ int generation = writer.getGeneration().getFull();
this.stringCache = cacheManager.getStringCache(generation);
this.templateCache = cacheManager.getTemplateCache(generation);
this.nodeCache = cacheManager.getNodeCache(generation);
@@ -989,7 +990,7 @@ public class DefaultSegmentWriter implem
try {
GCGeneration thatGen = id.getSegmentId().getGcGeneration();
GCGeneration thisGen = writer.getGeneration();
- return thatGen.compareWith(thisGen) < 0;
+ return thatGen.compareFull(thisGen) < 0 || thatGen.compareTail(thisGen) < 0;
} catch (SegmentNotFoundException snfe) {
// This SNFE means a defer compacted node state is too far
// in the past. It has been gc'ed already and cannot be
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/GCGeneration.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/GCGeneration.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/GCGeneration.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/GCGeneration.java Thu Jul 27 09:05:15 2017
@@ -22,25 +22,84 @@ import javax.annotation.Nonnull;
import com.google.common.base.Objects;
+/**
+ * Instances of this class represent the garbage collection generation related information
+ * of a segment. Each generation consists of a full and a tail part and a tail flag.
+ * The full and tail part are each increased by the respective garbage collection process.
+ * In the tail compaction case the segments written by the compactor will also have their
+ * tail flag set so cleanup can recognise them as not reclaimable (unless the full part is
+ * older then the number of retained generations). Segments written by normal repository
+ * writes will inherit the full and tail generations parts of the segments written by the
+ * previous compaction process. However the tail flag is never set for such segments ensuring
+ * cleanup after subsequent tail compactions can reclaim them once old enough (e.g. the tail
+ * part of the generation is older then the number of retained generations).
+ */
public final class GCGeneration {
- public static final GCGeneration NULL = new GCGeneration(0);
+ public static final GCGeneration NULL = new GCGeneration(0, 0, false);
- private final int generation;
+ private final int full;
+ private final int tail;
+ private final boolean isTail;
- public GCGeneration(int generation) {
- this.generation = generation;
+ public GCGeneration(int full, int tail, boolean isTail) {
+ this.full = full;
+ this.tail = tail;
+ this.isTail = isTail;
}
- public int getGeneration() {
- return generation;
+ public int getFull() {
+ return full;
}
- public GCGeneration next() {
- return new GCGeneration(generation + 1);
+ public int getTail() {
+ return tail;
}
- public int compareWith(@Nonnull GCGeneration that) {
- return generation - that.generation;
+ public boolean isTail() {
+ return isTail;
+ }
+
+ /**
+ * Create a new instance with the full part incremented by one and
+ * the tail part and the tail flag left unchanged.
+ */
+ @Nonnull
+ public GCGeneration nextFull() {
+ return new GCGeneration(full + 1, tail, false);
+ }
+
+ /**
+ * Create a new instance with the tail part incremented by one and
+ * the full part and the tail flag left unchanged.
+ * @return
+ */
+ @Nonnull
+ public GCGeneration nextTail() {
+ return new GCGeneration(full, tail + 1, true);
+ }
+
+ /**
+ * Create a new instance with the tail flag unset and the full
+ * part and tail part left unchanged.
+ * @return
+ */
+ @Nonnull
+ public GCGeneration nonTail() {
+ return new GCGeneration(full, tail, false);
+ }
+
+ /**
+ * The the difference of the full part between {@code this} and {@code that}.
+ */
+ public int compareFull(@Nonnull GCGeneration that) {
+ return full - that.full;
+ }
+
+ /**
+ * The the difference of the tail part between {@code this} and {@code that}.
+ */
+ public int compareTail(@Nonnull GCGeneration that) {
+ return tail - that.tail;
}
@Override
@@ -52,17 +111,20 @@ public final class GCGeneration {
return false;
}
GCGeneration that = (GCGeneration) other;
- return generation == that.generation;
+ return full == that.full && tail == that.tail && isTail == that.isTail;
}
@Override
public int hashCode() {
- return Objects.hashCode(generation);
+ return Objects.hashCode(full, tail, isTail);
}
@Override
public String toString() {
return "GCGeneration{" +
- "generation=" + generation + '}';
+ "full=" + full + ',' +
+ "tail=" + tail + ',' +
+ "isTail=" + isTail + '}';
}
+
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/Segment.java Thu Jul 27 09:05:15 2017
@@ -121,7 +121,9 @@ public class Segment {
*/
static final int BLOB_ID_SMALL_LIMIT = 1 << 12;
- static final int GC_GENERATION_OFFSET = 10;
+ static final int GC_TAIL_GENERATION_OFFSET = 4;
+
+ static final int GC_FULL_GENERATION_OFFSET = 10;
static final int REFERENCED_SEGMENT_ID_COUNT_OFFSET = 14;
@@ -373,9 +375,13 @@ public class Segment {
*/
@Nonnull
public static GCGeneration getGcGeneration(ByteBuffer data, UUID segmentId) {
- return isDataSegmentId(segmentId.getLeastSignificantBits())
- ? new GCGeneration(data.getInt(GC_GENERATION_OFFSET))
- : GCGeneration.NULL;
+ if (isDataSegmentId(segmentId.getLeastSignificantBits())) {
+ int full = data.getInt(GC_FULL_GENERATION_OFFSET);
+ int tail = data.getInt(GC_TAIL_GENERATION_OFFSET);
+ return new GCGeneration(full, tail & 0x7fffffff, tail < 0);
+ } else {
+ return GCGeneration.NULL;
+ }
}
/**
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBufferWriter.java Thu Jul 27 09:05:15 2017
@@ -27,7 +27,8 @@ import static com.google.common.collect.
import static java.lang.System.arraycopy;
import static java.lang.System.currentTimeMillis;
import static java.lang.System.identityHashCode;
-import static org.apache.jackrabbit.oak.segment.Segment.GC_GENERATION_OFFSET;
+import static org.apache.jackrabbit.oak.segment.Segment.GC_FULL_GENERATION_OFFSET;
+import static org.apache.jackrabbit.oak.segment.Segment.GC_TAIL_GENERATION_OFFSET;
import static org.apache.jackrabbit.oak.segment.Segment.HEADER_SIZE;
import static org.apache.jackrabbit.oak.segment.Segment.RECORD_ID_BYTES;
import static org.apache.jackrabbit.oak.segment.Segment.RECORD_SIZE;
@@ -182,10 +183,22 @@ public class SegmentBufferWriter impleme
buffer[4] = 0; // reserved
buffer[5] = 0; // reserved
- buffer[GC_GENERATION_OFFSET] = (byte) (generation.getGeneration() >> 24);
- buffer[GC_GENERATION_OFFSET + 1] = (byte) (generation.getGeneration() >> 16);
- buffer[GC_GENERATION_OFFSET + 2] = (byte) (generation.getGeneration() >> 8);
- buffer[GC_GENERATION_OFFSET + 3] = (byte) generation.getGeneration();
+ int tail = generation.getTail();
+ if (generation.isTail()) {
+ // Set highest order bit to mark segment created by tail compaction
+ tail |= 0x80000000;
+ }
+ buffer[GC_TAIL_GENERATION_OFFSET] = (byte) (tail >> 24);
+ buffer[GC_TAIL_GENERATION_OFFSET + 1] = (byte) (tail >> 16);
+ buffer[GC_TAIL_GENERATION_OFFSET + 2] = (byte) (tail >> 8);
+ buffer[GC_TAIL_GENERATION_OFFSET + 3] = (byte) tail;
+
+ int full = generation.getFull();
+ buffer[GC_FULL_GENERATION_OFFSET] = (byte) (full >> 24);
+ buffer[GC_FULL_GENERATION_OFFSET + 1] = (byte) (full >> 16);
+ buffer[GC_FULL_GENERATION_OFFSET + 2] = (byte) (full >> 8);
+ buffer[GC_FULL_GENERATION_OFFSET + 3] = (byte) full;
+
length = 0;
position = buffer.length;
recordNumbers = new MutableRecordNumbers();
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java Thu Jul 27 09:05:15 2017
@@ -28,6 +28,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreGCMonitor;
+// FIXME OAK-3349 add means to trigger full / tail compaction
public class SegmentRevisionGCMBean
extends AnnotatedStandardMBean
implements SegmentRevisionGC {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java Thu Jul 27 09:05:15 2017
@@ -233,7 +233,8 @@ public abstract class AbstractFileStore
long msb = id.getMostSignificantBits();
long lsb = id.getLeastSignificantBits();
ByteBuffer buffer = ByteBuffer.wrap(data);
- int generation = Segment.getGcGeneration(buffer, id).getGeneration();
+ // FIXME OAK-3349 also handle the tail part of the gc generation and flag during recovery
+ int generation = Segment.getGcGeneration(buffer, id).getFull();
w.recoverEntry(msb, lsb, data, 0, data.length, generation);
if (SegmentId.isDataSegmentId(lsb)) {
Segment segment = new Segment(tracker, segmentReader, tracker.newSegmentId(msb, lsb), buffer);
@@ -249,8 +250,9 @@ public abstract class AbstractFileStore
}
}
+ // FIXME OAK-3349 also handle the tail part of the gc generation and flag during recovery
private static void populateTarBinaryReferences(final Segment segment, final EntryRecovery w) {
- final int generation = segment.getGcGeneration().getGeneration();
+ final int generation = segment.getGcGeneration().getFull();
final UUID id = segment.getSegmentId().asUUID();
segment.forEachRecord(new RecordConsumer() {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Thu Jul 27 09:05:15 2017
@@ -180,7 +180,7 @@ public class FileStore extends AbstractF
}
this.segmentWriter = defaultSegmentWriterBuilder("sys")
- .withGeneration(this::getGcGeneration)
+ .withGeneration(() -> getGcGeneration().nonTail())
.withWriterPool()
.with(builder.getCacheManager()
.withAccessTracking("WRITE", builder.getStatsProvider()))
@@ -490,7 +490,8 @@ public class FileStore extends AbstractF
}
segment = new Segment(tracker, segmentReader, id, data);
- generation = segment.getGcGeneration().getGeneration();
+ // FIXME OAK-3349 also handle the tail part of the gc generation and flag when writing segments
+ generation = segment.getGcGeneration().getFull();
references = readReferences(segment);
binaryReferences = readBinaryReferences(segment);
}
@@ -659,9 +660,18 @@ public class FileStore extends AbstractF
return CompactionResult.succeeded(generation, gcOptions, compactedRootId);
}
- @Nonnull
+ @CheckForNull
+ private SegmentNodeState getBase() {
+ String root = gcJournal.read().getRoot();
+ RecordId rootId = RecordId.fromString(tracker, root);
+ return RecordId.NULL.equals(rootId)
+ ? null // FIXME OAK-3349 if no previous compacted base is found we fall back to full compaction by returning null. Add an respective log statement
+ : segmentReader.readNode(rootId); // FIXME OAK-3349 guard against SNFE and against rebasing onto a non compactor written state in case someone tampered with the journal.log. Add logging.
+ }
+
synchronized CompactionResult compact() {
- final GCGeneration newGeneration = getGcGeneration().next();
+ // FIXME OAK-3349 the generation needs to reflect the type of compaction: tail or full. Additionally we need to handler the graceful degradation case should #getBase() return null where we would fall back from a tail compaction to a full compaction.
+ final GCGeneration newGeneration = getGcGeneration().nextFull();
try {
Stopwatch watch = Stopwatch.createStarted();
gcListener.info("TarMK GC #{}: compaction started, gc options={}", GC_COUNT, gcOptions);
@@ -682,7 +692,7 @@ public class FileStore extends AbstractF
OnlineCompactor compactor = new OnlineCompactor(
segmentReader, writer, getBlobStore(), cancel, compactionMonitor::onNode);
- SegmentNodeState after = compact(null, before, compactor, writer);
+ SegmentNodeState after = compact(getBase(), before, compactor, writer);
if (after == null) {
gcListener.warn("TarMK GC #{}: compaction cancelled: {}.", GC_COUNT, cancel);
return compactionAborted(newGeneration);
@@ -1211,10 +1221,12 @@ public class FileStore extends AbstractF
return new Predicate<GCGeneration>() {
@Override
public boolean apply(GCGeneration generation) {
- return reference.compareWith(generation) >= retainedGenerations;
+ return reference.compareFull(generation) >= retainedGenerations
+ || reference.compareTail(generation) >= retainedGenerations;
}
@Override
public String toString() {
+ // FIXME OAK-3349 align string representation with above predicate
return "(" + reference + " - generation >= " + retainedGenerations + ")";
}
};
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Thu Jul 27 09:05:15 2017
@@ -102,14 +102,16 @@ public class FileStoreBuilder {
public void compactionSucceeded(@Nonnull GCGeneration newGeneration) {
compacted();
if (cacheManager != null) {
- cacheManager.evictOldGeneration(newGeneration.getGeneration());
+ // FIXME OAK-3349 also handle the tail part of the gc generation and flag. See also the respective todo at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter.SegmentWriteOperation.with()
+ cacheManager.evictOldGeneration(newGeneration.getFull());
}
}
@Override
public void compactionFailed(@Nonnull GCGeneration failedGeneration) {
if (cacheManager != null) {
- cacheManager.evictGeneration(failedGeneration.getGeneration());
+ // FIXME OAK-3349 also handle the tail part of the gc generation and flag. See also the respective todo at org.apache.jackrabbit.oak.segment.DefaultSegmentWriter.SegmentWriteOperation.with()
+ cacheManager.evictGeneration(failedGeneration.getFull());
}
}
};
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java Thu Jul 27 09:05:15 2017
@@ -43,13 +43,13 @@ import org.apache.jackrabbit.oak.segment
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+// FIXME OAK-3349 incorporate tail compaction information into the gc.log file and reflect through this class.
/**
* Persists the repository size and the reclaimed size following a cleanup
* operation in the {@link #GC_JOURNAL gc journal} file with the format:
* 'repoSize, reclaimedSize, timestamp, gcGen, nodes compacted'.
*/
public class GCJournal {
-
private static final Logger LOG = LoggerFactory.getLogger(GCJournal.class);
public static final String GC_JOURNAL = "gc.log";
@@ -178,7 +178,8 @@ public class GCJournal {
if (root == null) {
root = RecordId.NULL.toString10();
}
- return new GCJournalEntry(repoSize, reclaimedSize, ts, new GCGeneration(gcGen), nodes, root);
+ // FIXME OAK-3349 set tail part once we have that information in the gc.log file
+ return new GCJournalEntry(repoSize, reclaimedSize, ts, new GCGeneration(gcGen, 0, false), nodes, root);
}
@CheckForNull
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java Thu Jul 27 09:05:15 2017
@@ -692,7 +692,8 @@ class TarReader implements Closeable {
}
for (Entry<Integer, Map<UUID, Set<String>>> entry : generations.entrySet()) {
- if (skipGeneration.apply(new GCGeneration(entry.getKey()))) {
+ // FIXME OAK-3349 cannot properly clean up under tail compaction here since we don't have access to the tail part of the generation (i.e. it is not in the tar index).
+ if (skipGeneration.apply(new GCGeneration(entry.getKey(), 0, false))) {
continue;
}
@@ -748,7 +749,8 @@ class TarReader implements Closeable {
// CPU on subsequent look-ups.
TarEntry entry = entries[i];
UUID id = new UUID(entry.msb(), entry.lsb());
- if (context.shouldReclaim(id, new GCGeneration(entry.generation()), references.remove(id))) {
+ // FIXME OAK-3349 cannot properly clean up under tail compaction here since we don't have access to the tail part of the generation (i.e. it is not in the tar index). As a workaround for now the implementation of shouldReclaim() could directly read the tail generation from the segment (i.e. SegmentIdProvider.newSegmentId(id.getMostSignificantBits(), id.getLeastSignificantBits()).getGcGeneration().getTail())
+ if (context.shouldReclaim(id, new GCGeneration(entry.generation(), 0, false), references.remove(id))) {
reclaimable.add(id);
} else {
for (UUID refId : getReferences(id, graph)) {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyClientSync.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyClientSync.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyClientSync.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/standby/client/StandbyClientSync.java Thu Jul 27 09:05:15 2017
@@ -157,7 +157,7 @@ public final class StandbyClientSync imp
new StandbyClientSyncExecution(fileStore, client, newRunningSupplier()).execute();
GCGeneration genAfter = headGeneration(fileStore);
- if (autoClean && (genAfter.compareWith(genBefore)) > 0) {
+ if (autoClean && (genAfter.compareFull(genBefore)) > 0 || genAfter.compareTail(genBefore) > 0) {
log.info("New head generation detected (prevHeadGen: {} newHeadGen: {}), running cleanup.", genBefore, genAfter);
cleanupAndRemove();
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CacheWeightEstimator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CacheWeightEstimator.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CacheWeightEstimator.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CacheWeightEstimator.java Thu Jul 27 09:05:15 2017
@@ -18,7 +18,7 @@
*/
package org.apache.jackrabbit.oak.segment;
-import static org.apache.jackrabbit.oak.segment.Segment.GC_GENERATION_OFFSET;
+import static org.apache.jackrabbit.oak.segment.Segment.GC_FULL_GENERATION_OFFSET;
import static org.apache.jackrabbit.oak.segment.SegmentVersion.LATEST_VERSION;
import java.nio.ByteBuffer;
@@ -313,10 +313,10 @@ public class CacheWeightEstimator {
buffer[5] = 0; // refcount
int generation = 0;
- buffer[GC_GENERATION_OFFSET] = (byte) (generation >> 24);
- buffer[GC_GENERATION_OFFSET + 1] = (byte) (generation >> 16);
- buffer[GC_GENERATION_OFFSET + 2] = (byte) (generation >> 8);
- buffer[GC_GENERATION_OFFSET + 3] = (byte) generation;
+ buffer[GC_FULL_GENERATION_OFFSET] = (byte) (generation >> 24);
+ buffer[GC_FULL_GENERATION_OFFSET + 1] = (byte) (generation >> 16);
+ buffer[GC_FULL_GENERATION_OFFSET + 2] = (byte) (generation >> 8);
+ buffer[GC_FULL_GENERATION_OFFSET + 3] = (byte) generation;
ByteBuffer data = ByteBuffer.wrap(buffer);
SegmentId id = randomSegmentId(false);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java Thu Jul 27 09:05:15 2017
@@ -1156,7 +1156,7 @@ public class CompactionAndCleanupIT {
builder.getChildNode("a").remove();
NodeState deferCompacted = nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
assertEquals(
- uncompacted.getSegment().getGcGeneration().next(),
+ uncompacted.getSegment().getGcGeneration().nextFull(),
((SegmentNodeState)deferCompacted).getSegment().getGcGeneration());
}
}
@@ -1168,7 +1168,7 @@ public class CompactionAndCleanupIT {
SegmentNodeState sns1 = (SegmentNodeState) node1;
SegmentNodeState sns2 = (SegmentNodeState) node2;
assertEquals("GC generation should be bumped by one " + path,
- sns1.getSegment().getGcGeneration().next(), sns2.getSegment().getGcGeneration());
+ sns1.getSegment().getGcGeneration().nextFull(), sns2.getSegment().getGcGeneration());
assertEquals("Nodes should have same stable id: " + path,
sns1.getStableId(), sns2.getStableId());
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactorTest.java Thu Jul 27 09:05:15 2017
@@ -53,7 +53,7 @@ public class CompactorTest {
init(store);
SegmentWriter writer = defaultSegmentWriterBuilder("c")
- .withGeneration(new GCGeneration(1))
+ .withGeneration(new GCGeneration(1, 0, false))
.build(memoryStore);
Compactor compactor = new Compactor(memoryStore.getReader(), writer,
memoryStore.getBlobStore(), Suppliers.ofInstance(false), defaultGCOptions());
@@ -78,7 +78,7 @@ public class CompactorTest {
NodeStore store = SegmentNodeStoreBuilders.builder(memoryStore).build();
SegmentWriter writer = defaultSegmentWriterBuilder("c")
- .withGeneration(new GCGeneration(1))
+ .withGeneration(new GCGeneration(1, 0, false))
.build(memoryStore);
Compactor compactor = new Compactor(memoryStore.getReader(), writer,
memoryStore.getBlobStore(), Suppliers.ofInstance(true), defaultGCOptions());
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/NodeRecordTest.java Thu Jul 27 09:05:15 2017
@@ -65,19 +65,19 @@ public class NodeRecordTest {
SegmentWriter writer;
writer = defaultSegmentWriterBuilder("1")
- .withGeneration(new GCGeneration(1))
+ .withGeneration(new GCGeneration(1, 0, false))
.build(store);
SegmentNodeState one = new SegmentNodeState(store.getReader(), writer, store.getBlobStore(), writer.writeNode(EmptyNodeState.EMPTY_NODE));
writer.flush();
writer = defaultSegmentWriterBuilder("2")
- .withGeneration(new GCGeneration(2))
+ .withGeneration(new GCGeneration(2, 0, false))
.build(store);
SegmentNodeState two = new SegmentNodeState(store.getReader(), writer, store.getBlobStore(), writer.writeNode(one));
writer.flush();
writer = defaultSegmentWriterBuilder("3")
- .withGeneration(new GCGeneration(3))
+ .withGeneration(new GCGeneration(3, 0, false))
.build(store);
SegmentNodeState three = new SegmentNodeState(store.getReader(), writer, store.getBlobStore(), writer.writeNode(two));
writer.flush();
@@ -108,7 +108,7 @@ public class NodeRecordTest {
.with(nodesOnlyCache())
.build(store);
- generation.set(new GCGeneration(1));
+ generation.set(new GCGeneration(1, 0, false));
// Write a new node with a non trivial template. This record will
// belong to generation 1.
@@ -121,7 +121,7 @@ public class NodeRecordTest {
SegmentNodeState base = new SegmentNodeState(store.getReader(), writer, store.getBlobStore(), baseId);
writer.flush();
- generation.set(new GCGeneration(2));
+ generation.set(new GCGeneration(2, 0, false));
// Compact that same record to generation 2.
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java Thu Jul 27 09:05:15 2017
@@ -52,6 +52,7 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+// FIXME OAK-3349 implement tail compaction tests
public class OnlineCompactorTest {
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
@@ -81,7 +82,7 @@ public class OnlineCompactorTest {
assertNotNull(compacted);
assertFalse(uncompacted == compacted);
assertEquals(uncompacted, compacted);
- assertEquals(uncompacted.getSegment().getGcGeneration().next(), compacted.getSegment().getGcGeneration());
+ assertEquals(uncompacted.getSegment().getGcGeneration().nextFull(), compacted.getSegment().getGcGeneration());
modifyTestContent(nodeStore);
NodeState modified = nodeStore.getRoot();
@@ -89,7 +90,7 @@ public class OnlineCompactorTest {
assertNotNull(compacted);
assertFalse(modified == compacted);
assertEquals(modified, compacted);
- assertEquals(uncompacted.getSegment().getGcGeneration().next(), compacted.getSegment().getGcGeneration());
+ assertEquals(uncompacted.getSegment().getGcGeneration().nextFull(), compacted.getSegment().getGcGeneration());
}
@Test
@@ -102,7 +103,7 @@ public class OnlineCompactorTest {
assertNotNull(compacted);
assertFalse(uncompacted == compacted);
assertEquals(uncompacted, compacted);
- assertEquals(uncompacted.getSegment().getGcGeneration().next(), compacted.getSegment().getGcGeneration());
+ assertEquals(uncompacted.getSegment().getGcGeneration().nextFull(), compacted.getSegment().getGcGeneration());
}
@Test
@@ -119,7 +120,7 @@ public class OnlineCompactorTest {
@Nonnull
private static OnlineCompactor createCompactor(FileStore fileStore, Supplier<Boolean> cancel) {
SegmentWriter writer = defaultSegmentWriterBuilder("c")
- .withGeneration(new GCGeneration(1))
+ .withGeneration(new GCGeneration(1, 0, false))
.build(fileStore);
return new OnlineCompactor(fileStore.getReader(), writer, fileStore.getBlobStore(), cancel, () -> {});
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java?rev=1803145&r1=1803144&r2=1803145&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java Thu Jul 27 09:05:15 2017
@@ -46,21 +46,21 @@ public class GcJournalTest {
File directory = segmentFolder.newFolder();
GCJournal gc = new GCJournal(directory);
- gc.persist(0, 100, new GCGeneration(1), 50, RecordId.NULL.toString10());
+ gc.persist(0, 100, new GCGeneration(1,0 , false), 50, RecordId.NULL.toString10());
GCJournalEntry e0 = gc.read();
assertEquals(100, e0.getRepoSize());
assertEquals(0, e0.getReclaimedSize());
assertEquals(50, e0.getNodes());
assertEquals(RecordId.NULL.toString10(), e0.getRoot());
- gc.persist(0, 250, new GCGeneration(2), 75, RecordId.NULL.toString());
+ gc.persist(0, 250, new GCGeneration(2, 0, false), 75, RecordId.NULL.toString());
GCJournalEntry e1 = gc.read();
assertEquals(250, e1.getRepoSize());
assertEquals(0, e1.getReclaimedSize());
assertEquals(75, e1.getNodes());
assertEquals(RecordId.NULL.toString(), e1.getRoot());
- gc.persist(50, 200, new GCGeneration(3), 90, "foo");
+ gc.persist(50, 200, new GCGeneration(3, 0, false), 90, "foo");
GCJournalEntry e2 = gc.read();
assertEquals(200, e2.getRepoSize());
assertEquals(50, e2.getReclaimedSize());
@@ -68,7 +68,7 @@ public class GcJournalTest {
assertEquals("foo", e2.getRoot());
// same gen
- gc.persist(75, 300, new GCGeneration(3), 125, "bar");
+ gc.persist(75, 300, new GCGeneration(3, 0, false), 125, "bar");
GCJournalEntry e3 = gc.read();
assertEquals(200, e3.getRepoSize());
assertEquals(50, e3.getReclaimedSize());