You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2024/01/25 19:17:04 UTC
(jackrabbit-oak) 47/50: OAK-10199 : added logic to skip non garbage documents
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch DetailedGC/OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 1675e01a31ee2d09a6c2b5cf739c151d0f9d288e
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Wed Jun 28 02:06:50 2023 +0530
OAK-10199 : added logic to skip non garbage documents
---
.../plugins/document/VersionGarbageCollector.java | 7 +-
.../document/VersionGarbageCollectorIT.java | 145 +--------------------
2 files changed, 6 insertions(+), 146 deletions(-)
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index cca73d8e4b..c7af41b2c4 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -788,9 +788,10 @@ public class VersionGarbageCollector {
}
// if we are already at last document of current timeStamp,
- // we need to reset fromId and check again
+ // we need to reset fromId & increment fromModified and check again
if (!foundDoc && !Objects.equals(fromId, MIN_ID_VALUE)) {
fromId = MIN_ID_VALUE;
+ fromModified = fromModified + SECONDS.toMillis(5);
foundDoc = true; // to run while loop again
}
}
@@ -940,7 +941,7 @@ public class VersionGarbageCollector {
this.ownHeadRevision = headRevision.getRevision(nodeStore.getClusterId());
}
- public void collectGarbage(final NodeDocument doc, final GCPhases phases) {
+ public void collectGarbage(final NodeDocument doc, final GCPhases phases, final Revision revision) {
detailedGCStats.documentRead();
monitor.info("Collecting Detailed Garbage for doc [{}]", doc.getId());
@@ -987,6 +988,8 @@ public class VersionGarbageCollector {
.filter(p -> !retainPropSet.contains(p))
.mapToInt(x -> {
updateOp.remove(x);
+ setModified(updateOp,revision);
+ setDeleted(updateOp, revision, false);
return 1;})
.sum();
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
index 44de2312c0..7c7dd48950 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollectorIT.java
@@ -35,13 +35,9 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
-<<<<<<< HEAD
import static java.util.List.of;
import static java.util.Objects.requireNonNull;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-=======
-import static java.util.Objects.requireNonNull;
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
import static java.util.stream.Collectors.toList;
import static java.util.stream.StreamSupport.stream;
import static org.apache.commons.lang3.reflect.FieldUtils.writeField;
@@ -54,7 +50,6 @@ import static org.apache.jackrabbit.oak.api.Type.NAME;
import static org.apache.jackrabbit.oak.api.Type.STRING;
import static org.apache.jackrabbit.oak.api.Type.STRINGS;
import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
-<<<<<<< HEAD
import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.assertBranchRevisionNotRemovedFromAllDocuments;
import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.assertBranchRevisionRemovedFromAllDocuments;
@@ -62,17 +57,13 @@ import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.enableDe
import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.enableDetailGCDryRun;
import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.mergedBranchCommit;
import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.unmergedBranchCommit;
-=======
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
+import static org.apache.jackrabbit.oak.plugins.document.Collection.SETTINGS;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NUM_REVS_THRESHOLD;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.PREV_SPLIT_FACTOR;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.setModified;
-<<<<<<< HEAD
import static org.apache.jackrabbit.oak.plugins.document.Revision.getCurrentTimestamp;
-=======
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
import static org.apache.jackrabbit.oak.plugins.document.Revision.newRevision;
import static org.apache.jackrabbit.oak.plugins.document.TestUtils.NO_BINARY;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP;
@@ -451,15 +442,9 @@ public class VersionGarbageCollectorIT {
long maxAge = 1; //hours
long delta = MINUTES.toMillis(10);
//1. Go past GC age and check no GC done as nothing deleted
-<<<<<<< HEAD
clock.waitUntil(getCurrentTimestamp() + maxAge);
VersionGCStats stats = gc(gc, maxAge, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
- VersionGCStats stats = gc.gc(maxAge, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
//Remove property
@@ -472,29 +457,17 @@ public class VersionGarbageCollectorIT {
//2. Check that a deleted property is not collected before maxAge
//Clock cannot move back (it moved forward in #1) so double the maxAge
clock.waitUntil(clock.getTime() + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(1, stats.deletedPropsCount);
assertEquals(1, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(1, stats.deletedPropsGCCount);
- assertEquals(1, stats.updatedDetailedGCDocsCount);
- assertEquals("1:/z", stats.oldestModifiedDocId);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
//4. Check that a revived property (deleted and created again) does not get gc
NodeBuilder b3 = store1.getRoot().builder();
@@ -506,13 +479,8 @@ public class VersionGarbageCollectorIT {
store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -549,17 +517,10 @@ public class VersionGarbageCollectorIT {
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
VersionGCStats stats = gc(gc, maxAge*2, HOURS);
assertEquals(50_000, stats.deletedPropsCount);
assertEquals(5_000, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
-=======
- VersionGCStats stats = gc.gc(maxAge*2, HOURS);
- assertEquals(50_000, stats.deletedPropsGCCount);
- assertEquals(5_000, stats.updatedDetailedGCDocsCount);
- assertNotEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
}
@Test
@@ -579,11 +540,7 @@ public class VersionGarbageCollectorIT {
// enable the detailed gc flag
writeField(gc, "detailedGCEnabled", true, true);
long maxAge = 1; //hours
-<<<<<<< HEAD
long delta = MINUTES.toMillis(20);
-=======
- long delta = TimeUnit.MINUTES.toMillis(20);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
//Remove property
NodeBuilder b2 = store1.getRoot().builder();
@@ -596,40 +553,20 @@ public class VersionGarbageCollectorIT {
}
store1.merge(b2, EmptyHook.INSTANCE, CommitInfo.EMPTY);
// increase the clock to create new revision for next batch
-<<<<<<< HEAD
clock.waitUntil(getCurrentTimestamp() + (k * 5));
-=======
- clock.waitUntil(Revision.getCurrentTimestamp() + (k * 5));
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
}
store1.runBackgroundOperations();
-<<<<<<< HEAD
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
VersionGCStats stats = gc(gc, maxAge, HOURS);
assertEquals(50_000, stats.deletedPropsCount);
-=======
-
- //2. Check that a deleted property is not collected before maxAge
- //Clock cannot move back (it moved forward in #1) so double the maxAge
- clock.waitUntil(clock.getTime() + delta);
- VersionGCStats stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
-
- //3. Check that deleted property does get collected post maxAge
- clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-
- stats = gc.gc(maxAge, HOURS);
- assertEquals(50_000, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(5_000, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@Test
-<<<<<<< HEAD
public void testGC_WithNoDeletedProps_And_MoreThan_10_000_DocWithDifferentRevision() throws Exception {
//1. Create nodes with properties
NodeBuilder b1 = store1.getRoot().builder();
@@ -671,8 +608,6 @@ public class VersionGarbageCollectorIT {
}
@Test
-=======
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
public void testGCDeletedPropsAlreadyGCed() throws Exception {
//1. Create nodes with properties
NodeBuilder b1 = store1.getRoot().builder();
@@ -702,17 +637,10 @@ public class VersionGarbageCollectorIT {
//2. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(10, stats.deletedPropsCount);
assertEquals(10, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(10, stats.deletedPropsGCCount);
- assertEquals(10, stats.updatedDetailedGCDocsCount);
- assertNotEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
//3. now reCreate those properties again
NodeBuilder b3 = store1.getRoot().builder();
@@ -734,25 +662,17 @@ public class VersionGarbageCollectorIT {
//4. Check that deleted property does get collected again
// increment the clock again by more than 2 hours + delta
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(10, stats.deletedPropsCount);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(10, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(10, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@Test
public void testGCDeletedPropsAfterSystemCrash() throws Exception {
-<<<<<<< HEAD
if (store1 != null) {
store1.dispose();
}
-=======
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
final FailingDocumentStore fds = new FailingDocumentStore(fixture.createDocumentStore(), 42) {
@Override
public void dispose() {}
@@ -813,13 +733,8 @@ public class VersionGarbageCollectorIT {
//4. Check that deleted property does get collected again
// increment the clock again by more than 2 hours + delta
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
VersionGCStats stats = gc(gc, maxAge*2, HOURS);
assertEquals(10, stats.deletedPropsCount);
-=======
- VersionGCStats stats = gc.gc(maxAge*2, HOURS);
- assertEquals(10, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(10, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}
@@ -840,15 +755,9 @@ public class VersionGarbageCollectorIT {
long maxAge = 1; //hours
long delta = MINUTES.toMillis(10);
//1. Go past GC age and check no GC done as nothing deleted
-<<<<<<< HEAD
clock.waitUntil(getCurrentTimestamp() + maxAge);
VersionGCStats stats = gc(gc, maxAge, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
- VersionGCStats stats = gc.gc(maxAge, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
//Remove property
@@ -863,13 +772,8 @@ public class VersionGarbageCollectorIT {
//2. Check that a deleted property is not collected before maxAge
//Clock cannot move back (it moved forward in #1) so double the maxAge
clock.waitUntil(clock.getTime() + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
//3. Check that deleted property does get collected post maxAge
@@ -886,18 +790,12 @@ public class VersionGarbageCollectorIT {
store1.merge(b4, EmptyHook.INSTANCE, CommitInfo.EMPTY);
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
}
@Test
-<<<<<<< HEAD
public void testGCDeletedLongPathProps() throws Exception {
//1. Create nodes with properties
NodeBuilder b1 = store1.getRoot().builder();
@@ -1066,8 +964,6 @@ public class VersionGarbageCollectorIT {
}
@Test
-=======
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
public void testGCDeletedPropsWhenModifiedConcurrently() throws Exception {
//1. Create nodes with properties
NodeBuilder b1 = store1.getRoot().builder();
@@ -1081,19 +977,11 @@ public class VersionGarbageCollectorIT {
// enable the detailed gc flag
writeField(gc, "detailedGCEnabled", true, true);
long maxAge = 1; //hours
-<<<<<<< HEAD
long delta = MINUTES.toMillis(10);
//1. Go past GC age and check no GC done as nothing deleted
clock.waitUntil(getCurrentTimestamp() + maxAge);
VersionGCStats stats = gc(gc, maxAge, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- long delta = TimeUnit.MINUTES.toMillis(10);
- //1. Go past GC age and check no GC done as nothing deleted
- clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
- VersionGCStats stats = gc.gc(maxAge, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
//Remove property
@@ -1107,17 +995,10 @@ public class VersionGarbageCollectorIT {
//2. Check that a deleted property is not collected before maxAge
//Clock cannot move back (it moved forward in #1) so double the maxAge
clock.waitUntil(clock.getTime() + delta);
-<<<<<<< HEAD
stats = gc(gc, maxAge*2, HOURS);
assertEquals(0, stats.deletedPropsCount);
assertEquals(0, stats.updatedDetailedGCDocsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId); // as GC hadn't run
-=======
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
- assertEquals(0, stats.updatedDetailedGCDocsCount);
- assertNull(stats.oldestModifiedDocId); // as GC hadn't run
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
@@ -1141,19 +1022,11 @@ public class VersionGarbageCollectorIT {
}
};
-<<<<<<< HEAD
VersionGarbageCollector gc = new VersionGarbageCollector(store1, gcSupport, true, false);
stats = gc.gc(maxAge*2, HOURS);
assertEquals(0, stats.updatedDetailedGCDocsCount);
assertEquals(0, stats.deletedPropsCount);
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
-=======
- VersionGarbageCollector gc = new VersionGarbageCollector(store1, gcSupport, true);
- stats = gc.gc(maxAge*2, HOURS);
- assertEquals(0, stats.updatedDetailedGCDocsCount);
- assertEquals(0, stats.deletedPropsGCCount);
- assertNotEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
}
@Test
@@ -1173,19 +1046,11 @@ public class VersionGarbageCollectorIT {
// enable the detailed gc flag
writeField(gc, "detailedGCEnabled", true, true);
long maxAge = 1; //hours
-<<<<<<< HEAD
long delta = MINUTES.toMillis(10);
//1. Go past GC age and check no GC done as nothing deleted
clock.waitUntil(getCurrentTimestamp() + maxAge);
VersionGCStats stats = gc(gc, maxAge, HOURS);
assertEquals(0, stats.deletedPropsCount);
-=======
- long delta = TimeUnit.MINUTES.toMillis(10);
- //1. Go past GC age and check no GC done as nothing deleted
- clock.waitUntil(Revision.getCurrentTimestamp() + maxAge);
- VersionGCStats stats = gc.gc(maxAge, HOURS);
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(0, stats.updatedDetailedGCDocsCount);
//Remove property
@@ -1223,22 +1088,14 @@ public class VersionGarbageCollectorIT {
}
};
-<<<<<<< HEAD
gcRef.set(new VersionGarbageCollector(store1, gcSupport, true, false));
-=======
- gcRef.set(new VersionGarbageCollector(store1, gcSupport, true));
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
//3. Check that deleted property does get collected post maxAge
clock.waitUntil(clock.getTime() + HOURS.toMillis(maxAge*2) + delta);
stats = gcRef.get().gc(maxAge*2, HOURS);
assertTrue(stats.canceled);
assertEquals(0, stats.updatedDetailedGCDocsCount);
-<<<<<<< HEAD
assertEquals(0, stats.deletedPropsCount);
-=======
- assertEquals(0, stats.deletedPropsGCCount);
->>>>>>> d3b73cd921 (OAK-10199 : added unit cases to handle concurrent prop update and escaped properties update)
assertEquals(MIN_ID_VALUE, stats.oldestModifiedDocId);
}