You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2014/04/03 12:02:35 UTC
svn commit: r1584333 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins:
blob/MarkSweepGarbageCollector.java document/DocumentNodeStore.java
document/DocumentNodeStoreService.java document/mongo/MongoBlobStore.java
Author: chetanm
Date: Thu Apr 3 10:02:34 2014
New Revision: 1584333
URL: http://svn.apache.org/r1584333
Log:
OAK-1639 - MarkSweepGarbageCollector improvements
-- Fixed log messages
-- Made the blob gc age configurable for DocumentNodeStore
-- Fixed an issue with GC logic which was using maxGCInterval as time
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Thu Apr 3 10:02:34 2014
@@ -20,6 +20,7 @@ import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
+import java.sql.Timestamp;
import java.util.ArrayDeque;
import java.util.Iterator;
import java.util.List;
@@ -31,6 +32,7 @@ import java.util.concurrent.ConcurrentLi
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -124,10 +126,6 @@ public class MarkSweepGarbageCollector i
/**
* Instantiates a new blob garbage collector.
- *
- * @param marker
- * @param blobStore
- * @throws IOException Signals that an I/O exception has occurred.
*/
public MarkSweepGarbageCollector(
BlobReferenceRetriever marker,
@@ -137,6 +135,15 @@ public class MarkSweepGarbageCollector i
this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT, true, TimeUnit.HOURS.toMillis(24));
}
+ public MarkSweepGarbageCollector(
+ BlobReferenceRetriever marker,
+ GarbageCollectableBlobStore blobStore,
+ Executor executor,
+ long maxLastModifiedInterval)
+ throws IOException {
+ this(marker, blobStore, executor, TEMP_DIR, DEFAULT_BATCH_COUNT, true, maxLastModifiedInterval);
+ }
+
@Override
public void collectGarbage() throws Exception {
markAndSweep();
@@ -297,6 +304,12 @@ public class MarkSweepGarbageCollector i
return batchCount;
}
+ private long getLastMaxModifiedTime(){
+ return maxLastModifiedInterval > 0 ?
+ System.currentTimeMillis() - maxLastModifiedInterval : 0;
+
+ }
+
/**
* Save batch to file.
*/
@@ -327,10 +340,7 @@ public class MarkSweepGarbageCollector i
public void run() {
try {
LOG.debug("Blob ids to be deleted {}", ids);
- boolean deleted =
- blobStore.deleteChunks(ids,
- (maxLastModifiedInterval > 0 ? System.currentTimeMillis()
- - maxLastModifiedInterval : 0));
+ boolean deleted = blobStore.deleteChunks(ids,getLastMaxModifiedTime());
if (!deleted) {
exceptionQueue.addAll(ids);
}
@@ -346,13 +356,12 @@ public class MarkSweepGarbageCollector i
*/
private void iterateNodeTree() throws IOException {
final BufferedWriter writer = Files.newWriter(fs.getMarkedRefs(), Charsets.UTF_8);
+ final AtomicInteger count = new AtomicInteger();
try {
marker.collectReferences(
new ReferenceCollector() {
private final List<String> idBatch = Lists.newArrayListWithCapacity(getBatchCount());
- private int count = 0;
-
private final boolean debugMode = LOG.isTraceEnabled();
@Override
@@ -375,7 +384,7 @@ public class MarkSweepGarbageCollector i
if (debugMode) {
LOG.trace("chunkId : {}",id);
}
- count++;
+ count.getAndIncrement();
}
if (!idBatch.isEmpty()) {
@@ -385,13 +394,11 @@ public class MarkSweepGarbageCollector i
} catch (Exception e) {
throw new RuntimeException("Error in retrieving references", e);
}
-
- LOG.info("Number of valid blob references marked under mark phase of " +
- "Blob garbage collection [{}]",count);
}
}
);
-
+ LOG.info("Number of valid blob references marked under mark phase of " +
+ "Blob garbage collection [{}]",count.get());
// sort the marked references
fs.sort(fs.getMarkedRefs());
} finally {
@@ -412,7 +419,7 @@ public class MarkSweepGarbageCollector i
try {
bufferWriter = new BufferedWriter(
new FileWriter(fs.getAvailableRefs()));
- Iterator<String> idsIter = blobStore.getAllChunkIds(maxLastModifiedInterval);
+ Iterator<String> idsIter = blobStore.getAllChunkIds(getLastMaxModifiedTime());
List<String> ids = Lists.newArrayList();
@@ -431,7 +438,8 @@ public class MarkSweepGarbageCollector i
// sort the file
fs.sort(fs.getAvailableRefs());
- LOG.debug("Ending retrieving all blobs : {}", blobsCount);
+ LOG.debug("Number of blobs present in BlobStore : [{}] which have " +
+ "been last modified before [{}]", blobsCount, timestampToString(getLastMaxModifiedTime()));
} finally {
IOUtils.closeQuietly(bufferWriter);
}
@@ -575,4 +583,11 @@ public class MarkSweepGarbageCollector i
throw new UnsupportedOperationException();
}
}
+
+ /**
+ * Provides a readable string for given timestamp
+ */
+ private static String timestampToString(long timestamp){
+ return (new Timestamp(timestamp) + "00").substring(0, 23);
+ }
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Thu Apr 3 10:02:34 2014
@@ -39,6 +39,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.ReadWriteLock;
@@ -1628,16 +1629,18 @@ public final class DocumentNodeStore
* supports garbage collection
*
* @return garbage collector of the BlobStore supports GC otherwise null
+ * @param blobGcMaxAgeInSecs
*/
@CheckForNull
- public MarkSweepGarbageCollector createBlobGarbageCollector() {
+ public MarkSweepGarbageCollector createBlobGarbageCollector(long blobGcMaxAgeInSecs) {
MarkSweepGarbageCollector blobGC = null;
if(blobStore instanceof GarbageCollectableBlobStore){
try {
blobGC = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(this),
(GarbageCollectableBlobStore) blobStore,
- executor);
+ executor,
+ TimeUnit.SECONDS.toMillis(blobGcMaxAgeInSecs));
} catch (IOException e) {
throw new RuntimeException("Error occurred while initializing " +
"the MarkSweepGarbageCollector",e);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java Thu Apr 3 10:02:34 2014
@@ -132,15 +132,22 @@ public class DocumentNodeStoreService {
private Whiteboard whiteboard;
- private static final long DEFAULT_VER_GC_MAX_AGE = TimeUnit.DAYS.toSeconds(1);
- public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
/**
* Revisions older than this time would be garbage collected
*/
+ private static final long DEFAULT_VER_GC_MAX_AGE = TimeUnit.DAYS.toSeconds(1);
+ public static final String PROP_VER_GC_MAX_AGE = "versionGcMaxAgeInSecs";
private long versionGcMaxAgeInSecs = DEFAULT_VER_GC_MAX_AGE;
public static final String PROP_REV_RECOVERY_INTERVAL = "lastRevRecoveryJobIntervalInSecs";
+ /**
+ * Blob modified before this time duration would be considered for Blob GC
+ */
+ private static final long DEFAULT_BLOB_GC_MAX_AGE = TimeUnit.HOURS.toMillis(24);
+ public static final String PROP_BLOB_GC_MAX_AGE = "blobGcMaxAgeInSecs";
+ private long blobGcMaxAgeInSecs = DEFAULT_BLOB_GC_MAX_AGE;
+
@Activate
protected void activate(ComponentContext context, Map<String, ?> config) throws Exception {
@@ -231,6 +238,7 @@ public class DocumentNodeStoreService {
@Modified
protected void modified(Map<String, ?> config){
versionGcMaxAgeInSecs = PropertiesUtil.toLong(config.get(PROP_VER_GC_MAX_AGE), DEFAULT_VER_GC_MAX_AGE);
+ blobGcMaxAgeInSecs = PropertiesUtil.toLong(config.get(PROP_BLOB_GC_MAX_AGE), DEFAULT_BLOB_GC_MAX_AGE);
}
@Deactivate
@@ -323,7 +331,7 @@ public class DocumentNodeStoreService {
BlobGarbageCollector gc = new BlobGarbageCollector() {
@Override
public void collectGarbage() throws Exception {
- store.createBlobGarbageCollector().collectGarbage();
+ store.createBlobGarbageCollector(blobGcMaxAgeInSecs).collectGarbage();
}
};
registrations.add(registerMBean(whiteboard, BlobGCMBean.class, new BlobGC(gc, executor),
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java?rev=1584333&r1=1584332&r2=1584333&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobStore.java Thu Apr 3 10:02:34 2014
@@ -231,6 +231,7 @@ public class MongoBlobStore extends Cach
collection.find(builder.get(), fields).hint(fields)
.addOption(Bytes.QUERYOPTION_SLAVEOK);
+ //TODO The cursor needs to be closed
return new AbstractIterator<String>() {
@Override
protected String computeNext() {