You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by st...@apache.org on 2016/02/12 17:33:18 UTC
svn commit: r1730062 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/document/mongo/
Author: stefanegli
Date: Fri Feb 12 16:33:18 2016
New Revision: 1730062
URL: http://svn.apache.org/viewvc?rev=1730062&view=rev
Log:
OAK-3975 : configurable journalGC batch size with default of 100
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1730062&r1=1730061&r2=1730062&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java Fri Feb 12 16:33:18 2016
@@ -230,6 +230,19 @@ public class DocumentNodeStoreService {
)
private static final String PROP_JOURNAL_GC_MAX_AGE_MILLIS = "journalGCMaxAge";
+ /**
+ * Batch size used during to lookup and delete journal entries during journalGC
+ */
+ private static final int DEFAULT_JOURNAL_GC_BATCH_SIZE = 100;
+ @Property (intValue = DEFAULT_JOURNAL_GC_BATCH_SIZE,
+ label = "Batch size used for journalGC",
+ description = "The journal gc queries the journal for entries older than configured to delete them. " +
+ "It does so in batches to speed up the process. The batch size can be configured via this " +
+ " property. The trade-off is between reducing number of operations with a larger batch size, " +
+ " and consuming more memory less memory with a smaller batch size."
+ )
+ public static final String PROP_JOURNAL_GC_BATCH_SIZE = "journalGcBatchSize";
+
private static final long MB = 1024 * 1024;
private static enum DocumentStoreType {
@@ -722,11 +735,14 @@ public class DocumentNodeStoreService {
DEFAULT_JOURNAL_GC_INTERVAL_MILLIS);
final long journalGCMaxAge = toLong(context.getProperties().get(PROP_JOURNAL_GC_MAX_AGE_MILLIS),
DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS);
+ final int journalGCBatchSize = toInteger(context.getProperties().get(PROP_JOURNAL_GC_BATCH_SIZE),
+ DEFAULT_JOURNAL_GC_BATCH_SIZE);
+
Runnable journalGCJob = new Runnable() {
@Override
public void run() {
- nodeStore.getJournalGarbageCollector().gc(journalGCMaxAge, TimeUnit.MILLISECONDS);
+ nodeStore.getJournalGarbageCollector().gc(journalGCMaxAge, journalGCBatchSize, TimeUnit.MILLISECONDS);
}
};
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java?rev=1730062&r1=1730061&r2=1730062&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java Fri Feb 12 16:33:18 2016
@@ -38,9 +38,6 @@ import com.google.common.base.Stopwatch;
*/
public class JournalGarbageCollector {
- //copied from VersionGarbageCollector:
- private static final int DELETE_BATCH_SIZE = 450;
-
private final DocumentStore ds;
private static final Logger log = LoggerFactory.getLogger(JournalGarbageCollector.class);
@@ -57,10 +54,11 @@ public class JournalGarbageCollector {
* @param unit the timeunit for maxRevisionAge
* @return the number of entries that have been removed
*/
- public int gc(long maxRevisionAge, TimeUnit unit) {
+ public int gc(long maxRevisionAge, int batchSize, TimeUnit unit) {
long maxRevisionAgeInMillis = unit.toMillis(maxRevisionAge);
if (log.isDebugEnabled()) {
- log.debug("gc: Journal garbage collection starts with maxAge: {} min.", TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis));
+ log.debug("gc: Journal garbage collection starts with maxAge: {} min., batch size: {}.",
+ TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis), batchSize);
}
Stopwatch sw = Stopwatch.createStarted();
@@ -103,13 +101,12 @@ public class JournalGarbageCollector {
while (true) {
String fromKey = JournalEntry.asId(new Revision(startPointer, 0, clusterNodeId, branch));
String toKey = JournalEntry.asId(new Revision(System.currentTimeMillis() - maxRevisionAgeInMillis, Integer.MAX_VALUE, clusterNodeId, branch));
- int limit = DELETE_BATCH_SIZE;
- List<JournalEntry> deletionBatch = ds.query(Collection.JOURNAL, fromKey, toKey, limit);
+ List<JournalEntry> deletionBatch = ds.query(Collection.JOURNAL, fromKey, toKey, batchSize);
if (deletionBatch.size() > 0) {
ds.remove(Collection.JOURNAL, asKeys(deletionBatch));
numDeleted += deletionBatch.size();
}
- if (deletionBatch.size() < limit) {
+ if (deletionBatch.size() < batchSize) {
if (!branch) {
// do the same for branches:
// this will start at the beginning again with branch set to true
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java?rev=1730062&r1=1730061&r2=1730062&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java Fri Feb 12 16:33:18 2016
@@ -186,15 +186,15 @@ public class JournalTest extends Abstrac
JournalGarbageCollector gc = new JournalGarbageCollector(ns1);
// first clean up
Thread.sleep(100); // OAK-2979 : wait 100ms before doing the cleanup
- gc.gc(1, TimeUnit.MILLISECONDS);
+ gc.gc(1, 150, TimeUnit.MILLISECONDS);
Thread.sleep(100); // sleep just quickly
- assertEquals(0, gc.gc(1, TimeUnit.DAYS));
- assertEquals(0, gc.gc(6, TimeUnit.HOURS));
- assertEquals(0, gc.gc(1, TimeUnit.HOURS));
- assertEquals(0, gc.gc(10, TimeUnit.MINUTES));
- assertEquals(0, gc.gc(1, TimeUnit.MINUTES));
- assertEquals(0, gc.gc(1, TimeUnit.SECONDS));
- assertEquals(0, gc.gc(1, TimeUnit.MILLISECONDS));
+ assertEquals(0, gc.gc(1, 150, TimeUnit.DAYS));
+ assertEquals(0, gc.gc(6, 150, TimeUnit.HOURS));
+ assertEquals(0, gc.gc(1, 150, TimeUnit.HOURS));
+ assertEquals(0, gc.gc(10, 150, TimeUnit.MINUTES));
+ assertEquals(0, gc.gc(1, 150, TimeUnit.MINUTES));
+ assertEquals(0, gc.gc(1, 150, TimeUnit.SECONDS));
+ assertEquals(0, gc.gc(1, 150, TimeUnit.MILLISECONDS));
// create some entries that can be deleted thereupon
mk1.commit("/", "+\"regular1\": {}", null, null);
@@ -202,16 +202,16 @@ public class JournalTest extends Abstrac
mk1.commit("/", "+\"regular3\": {}", null, null);
mk1.commit("/regular2", "+\"regular4\": {}", null, null);
Thread.sleep(100); // sleep 100millis
- assertEquals(0, gc.gc(5, TimeUnit.SECONDS));
- assertEquals(0, gc.gc(1, TimeUnit.MILLISECONDS));
+ assertEquals(0, gc.gc(5, 150, TimeUnit.SECONDS));
+ assertEquals(0, gc.gc(1, 150, TimeUnit.MILLISECONDS));
ns1.runBackgroundOperations();
mk1.commit("/", "+\"regular5\": {}", null, null);
ns1.runBackgroundOperations();
mk1.commit("/", "+\"regular6\": {}", null, null);
ns1.runBackgroundOperations();
Thread.sleep(100); // sleep 100millis
- assertEquals(0, gc.gc(5, TimeUnit.SECONDS));
- assertEquals(3, gc.gc(1, TimeUnit.MILLISECONDS));
+ assertEquals(0, gc.gc(5, 150, TimeUnit.SECONDS));
+ assertEquals(3, gc.gc(1, 150, TimeUnit.MILLISECONDS));
}
@Test
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java?rev=1730062&r1=1730061&r2=1730062&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java Fri Feb 12 16:33:18 2016
@@ -200,7 +200,7 @@ public class JournalIT extends AbstractJ
JournalGarbageCollector gc = new JournalGarbageCollector(ns1);
clock.getTimeIncreasing();
clock.getTimeIncreasing();
- gc.gc(0, TimeUnit.MILLISECONDS); // cleanup everything that might still be there
+ gc.gc(0, 150, TimeUnit.MILLISECONDS); // cleanup everything that might still be there
// create entries as parametrized:
for(int i=offset; i<size+offset; i++) {
@@ -210,7 +210,7 @@ public class JournalIT extends AbstractJ
ns1.runBackgroundOperations();
}
Thread.sleep(100); // sleep 100millis
- assertEquals(size, gc.gc(0, TimeUnit.MILLISECONDS)); // should now be able to clean up everything
+ assertEquals(size, gc.gc(0, 150, TimeUnit.MILLISECONDS)); // should now be able to clean up everything
}
protected DocumentMK createMK(int clusterId, int asyncDelay) {