You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2017/07/25 08:58:56 UTC
svn commit: r1802903 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/document/mongo/
Author: mreutegg
Date: Tue Jul 25 08:58:56 2017
New Revision: 1802903
URL: http://svn.apache.org/viewvc?rev=1802903&view=rev
Log:
OAK-6488: Move journal maxRevisionAge to DocumentNodeStore
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalDiffLoaderTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentMK.java Tue Jul 25 08:58:56 2017
@@ -21,6 +21,7 @@ import static com.google.common.base.Pre
import static com.google.common.base.Suppliers.memoize;
import static com.google.common.base.Suppliers.ofInstance;
import static org.apache.jackrabbit.oak.commons.PathUtils.concat;
+import static org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreService.DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS;
import static org.apache.jackrabbit.oak.plugins.document.util.MongoConnection.readConcernLevel;
import java.io.InputStream;
@@ -602,6 +603,7 @@ public class DocumentMK {
new JournalPropertyHandlerFactory();
private int updateLimit = UPDATE_LIMIT;
private int commitValueCacheSize = 10000;
+ private long maxRevisionAgeMillis = DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS;
private GCMonitor gcMonitor = new LoggingGCMonitor(
LoggerFactory.getLogger(VersionGarbageCollector.class));
@@ -1147,6 +1149,21 @@ public class DocumentMK {
return commitValueCacheSize;
}
+ public Builder setJournalGCMaxAge(long maxRevisionAgeMillis) {
+ this.maxRevisionAgeMillis = maxRevisionAgeMillis;
+ return this;
+ }
+
+ /**
+ * The maximum age for journal entries in milliseconds. Older entries
+ * are candidates for GC.
+ *
+ * @return maximum age for journal entries in milliseconds.
+ */
+ public long getJournalGCMaxAge() {
+ return maxRevisionAgeMillis;
+ }
+
public Builder setGCMonitor(@Nonnull GCMonitor gcMonitor) {
this.gcMonitor = checkNotNull(gcMonitor);
return this;
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Tue Jul 25 08:58:56 2017
@@ -569,7 +569,8 @@ public final class DocumentNodeStore
this, builder.createVersionGCSupport());
this.versionGarbageCollector.setStatisticsProvider(builder.getStatisticsProvider());
this.versionGarbageCollector.setGCMonitor(builder.getGCMonitor());
- this.journalGarbageCollector = new JournalGarbageCollector(this);
+ this.journalGarbageCollector = new JournalGarbageCollector(
+ this, builder.getJournalGCMaxAge());
this.referencedBlobs =
builder.createReferencedBlobs(this);
this.lastRevSeeker = builder.createMissingLastRevSeeker();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java Tue Jul 25 08:58:56 2017
@@ -263,7 +263,7 @@ public class DocumentNodeStoreService {
)
private static final String PROP_JOURNAL_GC_INTERVAL_MILLIS = "journalGCInterval";
- private static final long DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS = 6*60*60*1000; // default is 6hours
+ static final long DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS = 6*60*60*1000; // default is 6hours
@Property(longValue = DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS,
label = "Maximum Age of Journal Entries (millis)",
description = "Long value indicating max age (in milliseconds) that "
@@ -480,6 +480,7 @@ public class DocumentNodeStoreService {
boolean bundlingDisabled = toBoolean(prop(PROP_BUNDLING_DISABLED), DEFAULT_BUNDLING_DISABLED);
boolean prefetchExternalChanges = toBoolean(prop(PROP_PREFETCH_EXTERNAL_CHANGES), false);
int updateLimit = toInteger(prop(PROP_UPDATE_LIMIT), DocumentMK.UPDATE_LIMIT);
+ long journalGCMaxAge = toLong(context.getProperties().get(PROP_JOURNAL_GC_MAX_AGE_MILLIS), DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS);
DocumentMK.Builder mkBuilder =
new DocumentMK.Builder().
setStatisticsProvider(statisticsProvider).
@@ -516,7 +517,8 @@ public class DocumentNodeStoreService {
}
}).
setPrefetchExternalChanges(prefetchExternalChanges).
- setUpdateLimit(updateLimit);
+ setUpdateLimit(updateLimit).
+ setJournalGCMaxAge(journalGCMaxAge);
if (!Strings.isNullOrEmpty(persistentCache)) {
mkBuilder.setPersistentCache(persistentCache);
@@ -983,14 +985,11 @@ public class DocumentNodeStoreService {
private void registerJournalGC(final DocumentNodeStore nodeStore) {
long journalGCInterval = toLong(context.getProperties().get(PROP_JOURNAL_GC_INTERVAL_MILLIS),
DEFAULT_JOURNAL_GC_INTERVAL_MILLIS);
- final long journalGCMaxAge = toLong(context.getProperties().get(PROP_JOURNAL_GC_MAX_AGE_MILLIS),
- DEFAULT_JOURNAL_GC_MAX_AGE_MILLIS);
Runnable journalGCJob = new Runnable() {
-
@Override
public void run() {
- nodeStore.getJournalGarbageCollector().gc(journalGCMaxAge, TimeUnit.MILLISECONDS);
+ nodeStore.getJournalGarbageCollector().gc();
}
};
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/JournalGarbageCollector.java Tue Jul 25 08:58:56 2017
@@ -28,6 +28,7 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Stopwatch;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
/**
@@ -55,37 +56,38 @@ public class JournalGarbageCollector {
private final DocumentNodeStore ns;
+ private final long maxRevisionAgeMillis;
+
private volatile long lastTailTimestampRefresh = Long.MIN_VALUE;
private Revision tailRevision;
- public JournalGarbageCollector(DocumentNodeStore nodeStore) {
+ public JournalGarbageCollector(DocumentNodeStore nodeStore,
+ long maxRevisionAgeMillis) {
this.ns = nodeStore;
+ this.maxRevisionAgeMillis = maxRevisionAgeMillis;
this.tailRevision = new Revision(0, 0, ns.getClusterId());
}
/**
- * Deletes entries in the journal that are older than the given
- * maxRevisionAge.
+ * Deletes entries in the journal that are older than
+ * {@link #getMaxRevisionAgeMillis()}.
*
- * @param maxRevisionAge entries older than this age will be removed
- * @param unit the {@linkplain TimeUnit} for maxRevisionAge
* @return the number of entries that have been removed
*/
- public int gc(long maxRevisionAge, TimeUnit unit) {
+ public int gc() {
DocumentStore ds = ns.getDocumentStore();
Revision keep = ns.getCheckpoints().getOldestRevisionToKeep();
- long maxRevisionAgeInMillis = unit.toMillis(maxRevisionAge);
long now = ns.getClock().getTime();
- long gcOlderThan = now - maxRevisionAgeInMillis;
+ long gcOlderThan = now - maxRevisionAgeMillis;
if (keep != null && keep.getTimestamp() < gcOlderThan) {
gcOlderThan = keep.getTimestamp();
log.debug("gc: Checkpoint {} is older than maxRevisionAge: {} min",
- keep, unit.toMinutes(maxRevisionAge));
+ keep, MILLISECONDS.toMinutes(maxRevisionAgeMillis));
}
if (log.isDebugEnabled()) {
log.debug("gc: Journal garbage collection starts with maxAge: {} min.",
- TimeUnit.MILLISECONDS.toMinutes(maxRevisionAgeInMillis));
+ MILLISECONDS.toMinutes(maxRevisionAgeMillis));
}
Stopwatch sw = Stopwatch.createStarted();
@@ -99,7 +101,7 @@ public class JournalGarbageCollector {
if (numDeleted > 0) {
log.info("gc: Journal garbage collection took {}, deleted {} entries that were older than {} min.",
- sw, numDeleted, TimeUnit.MILLISECONDS.toMinutes(now - gcOlderThan));
+ sw, numDeleted, MILLISECONDS.toMinutes(now - gcOlderThan));
}
return numDeleted;
}
@@ -115,6 +117,10 @@ public class JournalGarbageCollector {
return tailRevision;
}
+ long getMaxRevisionAgeMillis() {
+ return maxRevisionAgeMillis;
+ }
+
private void refreshTailRevisionIfNecessary() {
// refresh once a minute
long now = ns.getClock().getTime();
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalDiffLoaderTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalDiffLoaderTest.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalDiffLoaderTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalDiffLoaderTest.java Tue Jul 25 08:58:56 2017
@@ -270,7 +270,7 @@ public class JournalDiffLoaderTest {
ns1.runBackgroundOperations();
// collect journal entry created for /foo/nX
- ns1.getJournalGarbageCollector().gc(5, TimeUnit.MINUTES);
+ new JournalGarbageCollector(ns1, TimeUnit.MINUTES.toMillis(5)).gc();
// the next modification updates the root revision
// for clusterId 1 past the removed journal entry
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalGCTest.java Tue Jul 25 08:58:56 2017
@@ -65,6 +65,7 @@ public class JournalGCTest {
Clock c = new Clock.Virtual();
c.waitUntil(System.currentTimeMillis());
DocumentNodeStore ns = builderProvider.newBuilder()
+ .setJournalGCMaxAge(TimeUnit.HOURS.toMillis(1))
.clock(c).setAsyncDelay(0).getNodeStore();
// perform some change
@@ -85,7 +86,7 @@ public class JournalGCTest {
c.waitUntil(c.getTime() + TimeUnit.HOURS.toMillis(2));
// instruct journal collector to remove entries older than one hour
- ns.getJournalGarbageCollector().gc(1, TimeUnit.HOURS);
+ ns.getJournalGarbageCollector().gc();
// must not remove existing entry, because checkpoint is still valid
entry = ns.getDocumentStore().find(JOURNAL, JournalEntry.asId(head));
@@ -93,7 +94,7 @@ public class JournalGCTest {
ns.release(cp);
- ns.getJournalGarbageCollector().gc(1, TimeUnit.HOURS);
+ ns.getJournalGarbageCollector().gc();
// now journal GC can remove the entry
entry = ns.getDocumentStore().find(JOURNAL, JournalEntry.asId(head));
assertNull(entry);
@@ -104,6 +105,7 @@ public class JournalGCTest {
Clock c = new Clock.Virtual();
c.waitUntil(System.currentTimeMillis());
DocumentNodeStore ns = builderProvider.newBuilder()
+ .setJournalGCMaxAge(TimeUnit.HOURS.toMillis(1))
.clock(c).setAsyncDelay(0).getNodeStore();
JournalGarbageCollector jgc = ns.getJournalGarbageCollector();
@@ -114,7 +116,7 @@ public class JournalGCTest {
ns.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
ns.runBackgroundOperations();
- assertEquals(0, jgc.gc(1, TimeUnit.HOURS));
+ assertEquals(0, jgc.gc());
// current time, but without the increment done by getTime()
long now = c.getTime() - 1;
@@ -126,7 +128,7 @@ public class JournalGCTest {
c.waitUntil(c.getTime() + TimeUnit.HOURS.toMillis(1));
// must collect the journal entry created by the background update
- assertEquals(1, jgc.gc(1, TimeUnit.HOURS));
+ assertEquals(1, jgc.gc());
// current time, but without the increment done by getTime()
now = c.getTime() - 1;
@@ -230,7 +232,7 @@ public class JournalGCTest {
shouldWait.set(false);
// instruct journal GC to remove entries older than one hour - readingNs hasn't seen it
- writingNs.getJournalGarbageCollector().gc(1, TimeUnit.SECONDS);
+ new JournalGarbageCollector(writingNs, TimeUnit.SECONDS.toMillis(1)).gc();
// entry should be removed
JournalEntry entry = writingNs.getDocumentStore().find(JOURNAL, JournalEntry.asId(head));
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/JournalTest.java Tue Jul 25 08:58:56 2017
@@ -185,18 +185,17 @@ public class JournalTest extends Abstrac
DocumentNodeStore ns1 = mk1.getNodeStore();
// make sure we're visible and marked as active
ns1.renewClusterIdLease();
- JournalGarbageCollector gc = new JournalGarbageCollector(ns1);
// first clean up
Thread.sleep(100); // OAK-2979 : wait 100ms before doing the cleanup
- gc.gc(1, TimeUnit.MILLISECONDS);
+ jGC(ns1, 1, TimeUnit.MILLISECONDS);
Thread.sleep(100); // sleep just quickly
- assertEquals(0, gc.gc(1, TimeUnit.DAYS));
- assertEquals(0, gc.gc(6, TimeUnit.HOURS));
- assertEquals(0, gc.gc(1, TimeUnit.HOURS));
- assertEquals(0, gc.gc(10, TimeUnit.MINUTES));
- assertEquals(0, gc.gc(1, TimeUnit.MINUTES));
- assertEquals(0, gc.gc(1, TimeUnit.SECONDS));
- assertEquals(0, gc.gc(1, TimeUnit.MILLISECONDS));
+ assertEquals(0, jGC(ns1, 1, TimeUnit.DAYS));
+ assertEquals(0, jGC(ns1, 6, TimeUnit.HOURS));
+ assertEquals(0, jGC(ns1, 1, TimeUnit.HOURS));
+ assertEquals(0, jGC(ns1, 10, TimeUnit.MINUTES));
+ assertEquals(0, jGC(ns1, 1, TimeUnit.MINUTES));
+ assertEquals(0, jGC(ns1, 1, TimeUnit.SECONDS));
+ assertEquals(0, jGC(ns1, 1, TimeUnit.MILLISECONDS));
// create some entries that can be deleted thereupon
mk1.commit("/", "+\"regular1\": {}", null, null);
@@ -204,16 +203,16 @@ public class JournalTest extends Abstrac
mk1.commit("/", "+\"regular3\": {}", null, null);
mk1.commit("/regular2", "+\"regular4\": {}", null, null);
Thread.sleep(100); // sleep 100millis
- assertEquals(0, gc.gc(5, TimeUnit.SECONDS));
- assertEquals(0, gc.gc(1, TimeUnit.MILLISECONDS));
+ assertEquals(0, jGC(ns1, 5, TimeUnit.SECONDS));
+ assertEquals(0, jGC(ns1, 1, TimeUnit.MILLISECONDS));
ns1.runBackgroundOperations();
mk1.commit("/", "+\"regular5\": {}", null, null);
ns1.runBackgroundOperations();
mk1.commit("/", "+\"regular6\": {}", null, null);
ns1.runBackgroundOperations();
Thread.sleep(100); // sleep 100millis
- assertEquals(0, gc.gc(5, TimeUnit.SECONDS));
- assertEquals(3, gc.gc(1, TimeUnit.MILLISECONDS));
+ assertEquals(0, jGC(ns1, 5, TimeUnit.SECONDS));
+ assertEquals(3, jGC(ns1, 1, TimeUnit.MILLISECONDS));
}
@Test
@@ -471,6 +470,10 @@ public class JournalTest extends Abstrac
ns1.getDocumentStore().find(Collection.JOURNAL, id) != null);
}
+ private int jGC(DocumentNodeStore ns, long maxRevisionAge, TimeUnit unit) {
+ return new JournalGarbageCollector(ns, unit.toMillis(maxRevisionAge)).gc();
+ }
+
private DocumentMK createMK(int clusterId, int asyncDelay) {
if (ds == null) {
ds = new MemoryDocumentStore();
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java?rev=1802903&r1=1802902&r2=1802903&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/mongo/JournalIT.java Tue Jul 25 08:58:56 2017
@@ -18,7 +18,6 @@ package org.apache.jackrabbit.oak.plugin
import java.util.List;
import java.util.Set;
-import java.util.concurrent.TimeUnit;
import com.mongodb.DB;
@@ -194,10 +193,10 @@ public class JournalIT extends AbstractJ
DocumentNodeStore ns1 = mk1.getNodeStore();
// make sure we're visible and marked as active
renewClusterIdLease(ns1);
- JournalGarbageCollector gc = new JournalGarbageCollector(ns1);
+ JournalGarbageCollector gc = new JournalGarbageCollector(ns1, 0);
clock.getTimeIncreasing();
clock.getTimeIncreasing();
- gc.gc(0, TimeUnit.MILLISECONDS); // cleanup everything that might still be there
+ gc.gc(); // cleanup everything that might still be there
// create entries as parametrized:
for(int i=offset; i<size+offset; i++) {
@@ -207,7 +206,7 @@ public class JournalIT extends AbstractJ
ns1.runBackgroundOperations();
}
Thread.sleep(100); // sleep 100millis
- assertEquals(size, gc.gc(0, TimeUnit.MILLISECONDS)); // should now be able to clean up everything
+ assertEquals(size, gc.gc()); // should now be able to clean up everything
}
protected DocumentMK createMK(int clusterId, int asyncDelay) {