You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2017/06/28 11:04:03 UTC
svn commit: r1800150 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/file/
test/java/org/apache/jackrabbit/oak/segment/
Author: mduerig
Date: Wed Jun 28 11:04:03 2017
New Revision: 1800150
URL: http://svn.apache.org/viewvc?rev=1800150&view=rev
Log:
OAK-3349: Partial compaction
Generalise reclamation mechanism of binaries in the blob store using a predicate
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Wed Jun 28 11:04:03 2017
@@ -989,8 +989,8 @@ public class FileStore extends AbstractF
*/
synchronized void collectBlobReferences(ReferenceCollector collector) throws IOException {
segmentWriter.flush();
- int minGeneration = getGcGeneration() - gcOptions.getRetainedGenerations() + 1;
- tarFiles.collectBlobReferences(collector, minGeneration);
+ int oldGeneration = getGcGeneration() - gcOptions.getRetainedGenerations();
+ tarFiles.collectBlobReferences(collector, CompactionResult.newOldReclaimer(oldGeneration));
}
void cancel() {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarFiles.java Wed Jun 28 11:04:03 2017
@@ -668,9 +668,8 @@ class TarFiles implements Closeable {
return result;
}
- void collectBlobReferences(ReferenceCollector collector, int minGeneration) throws IOException {
+ void collectBlobReferences(ReferenceCollector collector, Predicate<Integer> reclaim) throws IOException {
Node head;
-
lock.writeLock().lock();
try {
if (writer != null) {
@@ -682,7 +681,7 @@ class TarFiles implements Closeable {
}
for (TarReader reader : iterable(head)) {
- reader.collectBlobReferences(collector, minGeneration);
+ reader.collectBlobReferences(collector, reclaim);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarReader.java Wed Jun 28 11:04:03 2017
@@ -716,12 +716,10 @@ class TarReader implements Closeable {
}
/**
- * Collect the references of those blobs that are reachable from any segment with a
- * generation at or above {@code minGeneration}.
- * @param collector
- * @param minGeneration
+ * Collect the references of those blobs that are reachable from any segment and
+ * are not reclaimable according to the {@code reclaim} predicate.
*/
- void collectBlobReferences(@Nonnull ReferenceCollector collector, int minGeneration) {
+ void collectBlobReferences(@Nonnull ReferenceCollector collector, Predicate<Integer> reclaim) {
Map<Integer, Map<UUID, Set<String>>> generations = getBinaryReferences();
if (generations == null) {
@@ -729,7 +727,7 @@ class TarReader implements Closeable {
}
for (Entry<Integer, Map<UUID, Set<String>>> entry : generations.entrySet()) {
- if (entry.getKey() < minGeneration) {
+ if (reclaim.apply(entry.getKey())) {
continue;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java?rev=1800150&r1=1800149&r2=1800150&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/CompactionAndCleanupIT.java Wed Jun 28 11:04:03 2017
@@ -20,6 +20,7 @@
package org.apache.jackrabbit.oak.segment;
import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Sets.newHashSet;
import static com.google.common.util.concurrent.Uninterruptibles.sleepUninterruptibly;
import static java.lang.Integer.getInteger;
import static java.lang.String.valueOf;
@@ -59,6 +60,8 @@ import java.util.concurrent.atomic.Atomi
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import javax.annotation.Nullable;
+
import com.google.common.io.ByteStreams;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
@@ -66,9 +69,12 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
+import org.apache.jackrabbit.oak.plugins.blob.datastore.OakFileDataStore;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreGCMonitor;
+import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.tool.Compact;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -1420,4 +1426,53 @@ public class CompactionAndCleanupIT {
builder.setProperty(UUID.randomUUID().toString(), UUID.randomUUID().toString());
}
}
+
+ private static BlobStore newBlobStore(File directory) {
+ OakFileDataStore delegate = new OakFileDataStore();
+ delegate.setPath(directory.getAbsolutePath());
+ delegate.init(null);
+ return new DataStoreBlobStore(delegate);
+ }
+
+
+ @Test
+ public void binaryRetentionWithDS()
+ throws IOException, InvalidFileStoreVersionException, CommitFailedException {
+ try (FileStore fileStore = fileStoreBuilder(new File(getFileStoreFolder(), "segmentstore"))
+ .withBlobStore(newBlobStore(new File(getFileStoreFolder(), "blobstore")))
+ .withGCOptions(defaultGCOptions().setGcSizeDeltaEstimation(0))
+ .build())
+ {
+ SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ builder.setProperty("bin", createBlob(nodeStore, 1000000));
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.flush();
+
+ class RefCollector implements ReferenceCollector {
+ final Set<String> references;
+
+ RefCollector(Set<String> references) {this.references = references;}
+
+ @Override
+ public void addReference(String reference, @Nullable String nodeId) {
+ references.add(reference);
+ }
+ }
+
+ Set<String> expectedReferences = newHashSet();
+ ReferenceCollector refCollector = new RefCollector(expectedReferences);
+ fileStore.collectBlobReferences(refCollector);
+
+ for(int k = 1; k <= 3; k++) {
+ fileStore.gc();
+ Set<String> actualReferences = newHashSet();
+ refCollector = new RefCollector(actualReferences);
+ fileStore.collectBlobReferences(refCollector);
+ assertEquals("Binary should be retained after " + k + "-th gc cycle",
+ expectedReferences, actualReferences);
+ }
+ }
+ }
}