You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2023/11/22 06:46:30 UTC
(jackrabbit-oak) 10/44: OAK-10199 : added test cases to fetch 5000+ modified docs in loop and verify them
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch DetailedGC/OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit ee42d6e0f2371c3ba2724aaa67244f26ac10520d
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Mon Jun 19 18:23:34 2023 +0530
OAK-10199 : added test cases to fetch 5000+ modified docs in loop and verify them
---
.../oak/plugins/document/VersionGCSupport.java | 2 +-
.../document/mongo/MongoVersionGCSupport.java | 2 +-
.../plugins/document/rdb/RDBVersionGCSupport.java | 3 +-
.../oak/plugins/document/VersionGCSupportTest.java | 115 ++++++++++++++++++++-
4 files changed, 114 insertions(+), 8 deletions(-)
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
index e58ec05903..db54553061 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupport.java
@@ -86,7 +86,7 @@ public class VersionGCSupport {
* @return matching documents.
*/
public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit,
- final String fromId) {
+ @NotNull final String fromId) {
return StreamSupport
.stream(getSelectedDocuments(store, MODIFIED_IN_SECS, 1, fromId).spliterator(), false)
.filter(input -> modifiedGreaterThanEquals(input, fromModified) && modifiedLessThan(input, toModified))
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
index 690fd5a0d6..9896857e36 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoVersionGCSupport.java
@@ -144,7 +144,7 @@ public class MongoVersionGCSupport extends VersionGCSupport {
*/
@Override
public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit,
- final String fromId) {
+ @NotNull final String fromId) {
// _modified >= fromModified && _modified < toModified && _id > fromId
final Bson query = and(gte(MODIFIED_IN_SECS, getModifiedInSecs(fromModified)),
lt(MODIFIED_IN_SECS, getModifiedInSecs(toModified)), gt(ID, fromId));
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
index 0d2f678911..7006c18683 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
@@ -48,6 +48,7 @@ import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.Unsupport
import org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.stats.Clock;
+import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -108,7 +109,7 @@ public class RDBVersionGCSupport extends VersionGCSupport {
*/
@Override
public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit,
- final String fromId) {
+ @NotNull final String fromId) {
List<QueryCondition> conditions = of(new QueryCondition(MODIFIED_IN_SECS, "<", getModifiedInSecs(toModified)),
new QueryCondition(MODIFIED_IN_SECS, ">=", getModifiedInSecs(fromModified)),
new QueryCondition(ID, ">", of(fromId)));
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java
index 4eb20986c2..0061771383 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCSupportTest.java
@@ -44,6 +44,8 @@ import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.ME
import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.MONGO;
import static org.apache.jackrabbit.oak.plugins.document.DocumentStoreFixture.RDB_H2;
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.NULL;
+import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.setModified;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.apache.jackrabbit.oak.stats.Clock.SIMPLE;
import static org.junit.Assert.assertEquals;
@@ -106,7 +108,7 @@ public class VersionGCSupportTest {
String id = getIdFromPath("/doc-" + i);
ids.add(id);
UpdateOp op = new UpdateOp(id, true);
- NodeDocument.setModified(op, r);
+ setModified(op, r);
NodeDocument.setDeleted(op, r, true);
store.create(NODES, of(op));
}
@@ -140,7 +142,7 @@ public class VersionGCSupportTest {
String id = getIdFromPath("/doc-modified" + i);
ids.add(id);
UpdateOp op = new UpdateOp(id, true);
- NodeDocument.setModified(op, r);
+ setModified(op, r);
store.create(NODES, of(op));
}
@@ -174,7 +176,7 @@ public class VersionGCSupportTest {
String id = getIdFromPath("/doc-del");
ids.add(id);
UpdateOp op = new UpdateOp(id, true);
- NodeDocument.setModified(op, r);
+ setModified(op, r);
NodeDocument.setDeleted(op, r, true);
store.create(NODES, of(op));
@@ -190,7 +192,7 @@ public class VersionGCSupportTest {
String id = getIdFromPath("/doc-modified");
ids.add(id);
UpdateOp op = new UpdateOp(id, true);
- NodeDocument.setModified(op, r);
+ setModified(op, r);
store.create(NODES, of(op));
NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE);
@@ -200,6 +202,110 @@ public class VersionGCSupportTest {
assertEquals(id, oldestModifiedDocId);
}
+ @Test
+ public void findModifiedDocsWhenModifiedIsDifferent() {
+ long secs = 42;
+ long offset = SECONDS.toMillis(secs);
+ List<UpdateOp> updateOps = new ArrayList<>(5_001);
+ for (int i = 0; i < 5_001; i++) {
+ Revision r = new Revision(offset + (i * 5), 0, 1);
+ String id = getIdFromPath("/x" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ updateOps.add(op);
+ }
+ // create 5_000 nodes
+ store.create(NODES, updateOps);
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE);
+ String oldestModifiedDocId = oldestModifiedDoc.getId();
+ long oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ assertEquals(40L, oldestModifiedDocTs);
+ assertEquals("1:/x0", oldestModifiedDocId);
+
+ for(int i = 0; i < 5; i++) {
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), Long.MAX_VALUE, 1000, oldestModifiedDocId);
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ long count = stream(modifiedDocs.spliterator(), false).count();
+ assertEquals(1000, count);
+ for (NodeDocument modifiedDoc : modifiedDocs) {
+ oldestModifiedDoc = modifiedDoc;
+ }
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ }
+ }
+
+ @Test
+ public void findModifiedDocsWhenOldestDocIsPresent() {
+ long offset = SECONDS.toMillis(42);
+ List<UpdateOp> updateOps = new ArrayList<>(5_001);
+ for (int i = 0; i < 5_001; i++) {
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/x" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ updateOps.add(op);
+ }
+ // create 5_000 nodes
+ store.create(NODES, updateOps);
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE);
+ String oldestModifiedDocId = oldestModifiedDoc.getId();
+ long oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ assertEquals(40L, oldestModifiedDocTs);
+ assertEquals("1:/x0", oldestModifiedDocId);
+
+ for(int i = 0; i < 5; i++) {
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), Long.MAX_VALUE, 1000, oldestModifiedDocId);
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ long count = stream(modifiedDocs.spliterator(), false).count();
+ assertEquals(1000, count);
+ for (NodeDocument modifiedDoc : modifiedDocs) {
+ oldestModifiedDoc = modifiedDoc;
+ }
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ }
+ }
+
+ @Test
+ public void findModifiedDocsWhenOldestDocIsAbsent() {
+
+ NodeDocument oldestModifiedDoc = gcSupport.getOldestModifiedDoc(SIMPLE);
+ String oldestModifiedDocId = MIN_ID_VALUE;
+ long oldestModifiedDocTs = 0L;
+ assertEquals(NULL, oldestModifiedDoc);
+
+ long offset = SECONDS.toMillis(42);
+ List<UpdateOp> updateOps = new ArrayList<>(5_000);
+ for (int i = 0; i < 5_000; i++) {
+ Revision r = new Revision(offset, 0, 1);
+ String id = getIdFromPath("/x" + i);
+ ids.add(id);
+ UpdateOp op = new UpdateOp(id, true);
+ setModified(op, r);
+ updateOps.add(op);
+ }
+ // create 5_000 nodes
+ store.create(NODES, updateOps);
+
+
+ for(int i = 0; i < 5; i++) {
+ Iterable<NodeDocument> modifiedDocs = gcSupport.getModifiedDocs(SECONDS.toMillis(oldestModifiedDocTs), Long.MAX_VALUE, 1000, oldestModifiedDocId);
+ assertTrue(isInOrder(modifiedDocs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
+ long count = stream(modifiedDocs.spliterator(), false).count();
+ assertEquals(1000, count);
+ for (NodeDocument modifiedDoc : modifiedDocs) {
+ oldestModifiedDoc = modifiedDoc;
+ }
+ oldestModifiedDocId = oldestModifiedDoc.getId();
+ oldestModifiedDocTs = ofNullable(oldestModifiedDoc.getModified()).orElse(0L);
+ }
+ }
+
private void assertPossiblyDeleted(long fromSeconds, long toSeconds, long num) {
Iterable<NodeDocument> docs = gcSupport.getPossiblyDeletedDocs(SECONDS.toMillis(fromSeconds), SECONDS.toMillis(toSeconds));
assertEquals(num, stream(docs.spliterator(), false).count());
@@ -207,7 +313,6 @@ public class VersionGCSupportTest {
private void assertModified(long fromSeconds, long toSeconds, long num) {
Iterable<NodeDocument> docs = gcSupport.getModifiedDocs(SECONDS.toMillis(fromSeconds), SECONDS.toMillis(toSeconds), 10, MIN_ID_VALUE);
- docs.forEach(d -> System.out.println(d.getModified() + " " + d.getId()));
assertEquals(num, stream(docs.spliterator(), false).count());
assertTrue(isInOrder(docs, (o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)));
}