You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by da...@apache.org on 2023/10/03 14:16:23 UTC
[jackrabbit-oak] 28/29: OAK-10199 : removed DetailedGC code changes for RDBDocumentStore
This is an automated email from the ASF dual-hosted git repository.
daim pushed a commit to branch DetailedGC/OAK-10199
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 7f3511f42f0c7a18486f7ea445bde0ea8794e30a
Author: Rishabh Kumar <di...@adobe.com>
AuthorDate: Wed Aug 9 00:08:12 2023 +0530
OAK-10199 : removed DetailedGC code changes for RDBDocumentStore
---
.../plugins/document/VersionGarbageCollector.java | 4 +-
.../oak/plugins/document/rdb/RDBDocumentStore.java | 16 +---
.../plugins/document/rdb/RDBDocumentStoreJDBC.java | 16 ++--
.../plugins/document/rdb/RDBVersionGCSupport.java | 104 ---------------------
.../oak/plugins/document/VersionGCInitTest.java | 13 +--
5 files changed, 18 insertions(+), 135 deletions(-)
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 8f2c977274..a32ded6c5b 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -350,7 +350,7 @@ public class VersionGarbageCollector {
return "VersionGCStats{" +
"ignoredGCDueToCheckPoint=" + ignoredGCDueToCheckPoint +
- "ignoredDetailedGCDueToCheckPoint=" + ignoredDetailedGCDueToCheckPoint +
+ ", ignoredDetailedGCDueToCheckPoint=" + ignoredDetailedGCDueToCheckPoint +
", canceled=" + canceled +
", deletedDocGCCount=" + deletedDocGCCount + " (of which leaf: " + deletedLeafDocGCCount + ")" +
", updateResurrectedGCCount=" + updateResurrectedGCCount +
@@ -950,7 +950,7 @@ public class VersionGarbageCollector {
updatedDocs = (int) oldDocs.stream().filter(Objects::nonNull).count();
stats.updatedDetailedGCDocsCount += updatedDocs;
stats.deletedPropsGCCount += deletedProps;
- log.info("Updated [{}] documents, deleted [{}] properties", updatedDocs, deletedProps);
+ log.debug("Updated [{}] documents, deleted [{}] properties", updatedDocs, deletedProps);
// now reset delete metadata
updateOpList.clear();
deletedPropsCountMap.clear();
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
index 25a3aca5e7..82c09e213d 100755
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStore.java
@@ -971,8 +971,8 @@ public class RDBDocumentStore implements DocumentStore {
public static String VERSIONPROP = "__version";
// set of supported indexed properties
- private static final Set<String> INDEXEDPROPERTIES = new HashSet<>(Arrays.asList(MODIFIED,
- NodeDocument.HAS_BINARY_FLAG, NodeDocument.DELETED_ONCE, NodeDocument.SD_TYPE, NodeDocument.SD_MAX_REV_TIME_IN_SECS, VERSIONPROP, ID));
+ private static final Set<String> INDEXEDPROPERTIES = new HashSet<String>(Arrays.asList(new String[] { MODIFIED,
+ NodeDocument.HAS_BINARY_FLAG, NodeDocument.DELETED_ONCE, NodeDocument.SD_TYPE, NodeDocument.SD_MAX_REV_TIME_IN_SECS, VERSIONPROP }));
// set of required table columns
private static final Set<String> REQUIREDCOLUMNS = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(
@@ -1840,7 +1840,7 @@ public class RDBDocumentStore implements DocumentStore {
}
protected <T extends Document> Iterable<T> queryAsIterable(final Collection<T> collection, String fromKey, String toKey,
- final List<String> excludeKeyPatterns, final List<QueryCondition> conditions, final int limit, final List<String> sortBy) {
+ final List<String> excludeKeyPatterns, final List<QueryCondition> conditions, final int limit, final String sortBy) {
final RDBTableMetaData tmd = getTable(collection);
Set<String> allowedProps = Sets.intersection(INDEXEDPROPERTIES, tmd.getColumnProperties());
@@ -1853,16 +1853,6 @@ public class RDBDocumentStore implements DocumentStore {
}
}
- if (sortBy != null && !sortBy.isEmpty()) {
- for (String key : sortBy) {
- if (!allowedProps.contains(key)) {
- final String message = "indexed property " + key + " not supported. supported properties are " + allowedProps;
- LOG.error(message);
- throw new UnsupportedIndexedPropertyException(message);
- }
- }
- }
-
final String from = collection == Collection.NODES && NodeDocument.MIN_ID_VALUE.equals(fromKey) ? null : fromKey;
final String to = collection == Collection.NODES && NodeDocument.MAX_ID_VALUE.equals(toKey) ? null : toKey;
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
index 87d0f4b4dd..5caa65d875 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBDocumentStoreJDBC.java
@@ -16,11 +16,8 @@
*/
package org.apache.jackrabbit.oak.plugins.document.rdb;
-import static java.util.List.of;
-import static java.util.stream.Collectors.joining;
import static org.apache.jackrabbit.guava.common.collect.Iterables.transform;
import static org.apache.jackrabbit.guava.common.collect.Sets.newHashSet;
-import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.CHAR2OCTETRATIO;
import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.asBytes;
import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBJDBCTools.asDocumentStoreException;
@@ -462,7 +459,7 @@ public class RDBDocumentStoreJDBC {
+ excludeKeyPatterns + ", conditions=" + conditions + ", limit=" + limit)
: null);
stmt = prepareQuery(connection, tmd, fields, minId,
- maxId, excludeKeyPatterns, conditions, limit, of(ID));
+ maxId, excludeKeyPatterns, conditions, limit, "ID");
rs = stmt.executeQuery();
while (rs.next() && result.size() < limit) {
int field = 1;
@@ -557,7 +554,7 @@ public class RDBDocumentStoreJDBC {
@NotNull
public Iterator<RDBRow> queryAsIterator(RDBConnectionHandler ch, RDBTableMetaData tmd, String minId, String maxId,
- List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, List<String> sortBy) throws SQLException {
+ List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, String sortBy) throws SQLException {
return new ResultSetIterator(ch, tmd, minId, maxId, excludeKeyPatterns, conditions, limit, sortBy);
}
@@ -576,7 +573,7 @@ public class RDBDocumentStoreJDBC {
private long pstart;
public ResultSetIterator(RDBConnectionHandler ch, RDBTableMetaData tmd, String minId, String maxId,
- List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, List<String> sortBy) throws SQLException {
+ List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, String sortBy) throws SQLException {
long start = System.currentTimeMillis();
try {
this.ch = ch;
@@ -698,7 +695,7 @@ public class RDBDocumentStoreJDBC {
@NotNull
private PreparedStatement prepareQuery(Connection connection, RDBTableMetaData tmd, String columns, String minId, String maxId,
- List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, List<String> sortBy) throws SQLException {
+ List<String> excludeKeyPatterns, List<QueryCondition> conditions, int limit, String sortBy) throws SQLException {
StringBuilder selectClause = new StringBuilder();
@@ -717,8 +714,8 @@ public class RDBDocumentStoreJDBC {
query.append(" where ").append(whereClause);
}
- if (sortBy != null && !sortBy.isEmpty()) {
- query.append(" order by ").append(sortBy.stream().map(INDEXED_PROP_MAPPING::get).collect(joining(", ")));
+ if (sortBy != null) {
+ query.append(" order by ID");
}
if (limit != Integer.MAX_VALUE) {
@@ -969,7 +966,6 @@ public class RDBDocumentStoreJDBC {
tmp.put(NodeDocument.SD_TYPE, "SDTYPE");
tmp.put(NodeDocument.SD_MAX_REV_TIME_IN_SECS, "SDMAXREVTIME");
tmp.put(RDBDocumentStore.VERSIONPROP, "VERSION");
- tmp.put(ID, "ID");
INDEXED_PROP_MAPPING = Collections.unmodifiableMap(tmp);
}
diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
index 27c582311b..a463499793 100644
--- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
+++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/rdb/RDBVersionGCSupport.java
@@ -16,23 +16,7 @@
*/
package org.apache.jackrabbit.oak.plugins.document.rdb;
-import static java.util.Comparator.comparing;
-import static java.util.List.of;
-import static java.util.Optional.empty;
-import static java.util.Optional.ofNullable;
-import static java.util.stream.Collectors.toList;
-import static java.util.stream.Stream.concat;
-import static java.util.stream.StreamSupport.stream;
import static org.apache.jackrabbit.guava.common.collect.Iterables.filter;
-import static org.apache.jackrabbit.guava.common.collect.Iterables.size;
-import static org.apache.jackrabbit.oak.plugins.document.Collection.NODES;
-import static org.apache.jackrabbit.oak.plugins.document.Document.ID;
-import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
-import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MODIFIED_IN_SECS;
-import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.getModifiedInSecs;
-import static org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.EMPTY_KEY_PATTERN;
-import static org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable.wrap;
-import static org.apache.jackrabbit.oak.plugins.document.util.Utils.closeIfCloseable;
import java.io.Closeable;
import java.io.IOException;
@@ -41,15 +25,11 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.function.Supplier;
-import java.util.stream.Stream;
import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
import org.apache.jackrabbit.oak.plugins.document.Collection;
-import org.apache.jackrabbit.oak.plugins.document.Document;
import org.apache.jackrabbit.oak.plugins.document.DocumentStoreException;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument.SplitDocType;
@@ -60,7 +40,6 @@ import org.apache.jackrabbit.oak.plugins.document.rdb.RDBDocumentStore.Unsupport
import org.apache.jackrabbit.oak.plugins.document.util.CloseableIterable;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.stats.Clock;
-import org.jetbrains.annotations.NotNull;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -106,67 +85,6 @@ public class RDBVersionGCSupport extends VersionGCSupport {
}
}
- /**
- * Returns documents that have a {@link NodeDocument#MODIFIED_IN_SECS} value
- * within the given range and are greater than given @{@link NodeDocument#ID}.
- * <p>
- * The two passed modified timestamps are in milliseconds
- * since the epoch and the implementation will convert them to seconds at
- * the granularity of the {@link NodeDocument#MODIFIED_IN_SECS} field and
- * then perform the comparison.
- *
- *
- * @param fromModified the lower bound modified timestamp (inclusive)
- * @param toModified the upper bound modified timestamp (exclusive)
- * @param limit the limit of documents to return
- * @param fromId the lower bound {@link NodeDocument#ID}
- * @return matching documents.
- */
- @Override
- public Iterable<NodeDocument> getModifiedDocs(final long fromModified, final long toModified, final int limit,
- @NotNull final String fromId) {
- // (_modified = fromModified && _id > fromId || _modified > fromModified && _modified < toModified)
- // TODO : introduce support for OR where clause in RDBDocumentStore
- final List<QueryCondition> c1 = of(new QueryCondition(MODIFIED_IN_SECS, "=", getModifiedInSecs(fromModified)),
- new QueryCondition(ID, ">", of(fromId)));
-
- final List<QueryCondition> c2 = of(new QueryCondition(MODIFIED_IN_SECS, "<", getModifiedInSecs(toModified)),
- new QueryCondition(MODIFIED_IN_SECS, ">", getModifiedInSecs(fromModified)));
-
- if (MODE == 1) {
- return getNodeDocuments(() -> getIterator(EMPTY_KEY_PATTERN, c1), () -> getIterator(EMPTY_KEY_PATTERN, c2), limit);
- } else {
- return getNodeDocuments(() -> store.queryAsIterable(NODES, null, null, EMPTY_KEY_PATTERN, c1, limit, of(MODIFIED_IN_SECS, ID)),
- () -> store.queryAsIterable(NODES, null, null, EMPTY_KEY_PATTERN, c2, limit, of(MODIFIED_IN_SECS, ID)),
- limit);
- }
- }
-
- /**
- * To fetch {@link NodeDocument} from database
- *
- * @param supplier1 document supplier on basis of 1st Condition
- * @param supplier2 document supplier on basis of 2nd Condition
- * @param limit no. of documents to fetch from db
- * @return sorted documents supplied by supplier1 & supplier2
- */
- private Iterable<NodeDocument> getNodeDocuments(final Supplier<Iterable<NodeDocument>> supplier1, final Supplier<Iterable<NodeDocument>> supplier2, final int limit) {
-
- final Iterable<NodeDocument> itr1 = supplier1.get();
- if (size(itr1) >= limit) {
- return itr1;
- }
-
- final Iterable<NodeDocument> itr2 = supplier2.get();
-
- final Stream<NodeDocument> s1 = stream(itr1.spliterator(), false);
- final Stream<NodeDocument> s2 = stream(itr2.spliterator(), false);
- return wrap(concat(s1, s2).sorted((o1, o2) -> comparing(NodeDocument::getModified).thenComparing(Document::getId).compare(o1, o2)).limit(limit).collect(toList()), () -> {
- closeIfCloseable(itr1);
- closeIfCloseable(itr2);
- });
- }
-
@Override
protected Iterable<NodeDocument> identifyGarbage(final Set<SplitDocType> gcTypes, final RevisionVector sweepRevs,
final long oldestRevTimeStamp) {
@@ -321,28 +239,6 @@ public class RDBVersionGCSupport extends VersionGCSupport {
}
}
- /**
- * Retrieve the time of the oldest modified document.
- *
- * @param clock System Clock
- * @return the timestamp of the oldest modified document.
- */
- @Override
- public Optional<NodeDocument> getOldestModifiedDoc(Clock clock) {
-
- Iterable<NodeDocument> modifiedDocs = null;
- try {
- modifiedDocs = getModifiedDocs(0L, clock.getTime(), 1, MIN_ID_VALUE);
- return modifiedDocs.iterator().hasNext() ? ofNullable(modifiedDocs.iterator().next()) : empty();
- } catch (DocumentStoreException ex) {
- LOG.error("getOldestModifiedDoc() <- Error ", ex);
- } finally {
- closeIfCloseable(modifiedDocs);
- }
- LOG.info("No Modified Doc has been found, retuning empty");
- return empty();
- }
-
@Override
public long getDeletedOnceCount() {
return store.queryCount(Collection.NODES, null, null, RDBDocumentStore.EMPTY_KEY_PATTERN,
diff --git a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
index 0c6b2fccdf..eec3e33b6b 100644
--- a/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
+++ b/oak-store-document/src/test/java/org/apache/jackrabbit/oak/plugins/document/VersionGCInitTest.java
@@ -31,6 +31,7 @@ import static org.apache.jackrabbit.oak.plugins.document.DetailGCHelper.enableDe
import static org.apache.jackrabbit.oak.plugins.document.NodeDocument.MIN_ID_VALUE;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP;
import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP;
+import static org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.SETTINGS_COLLECTION_ID;
import static org.apache.jackrabbit.oak.plugins.document.util.Utils.getIdFromPath;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -51,12 +52,12 @@ public class VersionGCInitTest {
@Test
public void lazyInitialize() throws Exception {
DocumentStore store = ns.getDocumentStore();
- Document vgc = store.find(SETTINGS, "versionGC");
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNull(vgc);
ns.getVersionGarbageCollector().gc(1, DAYS);
- vgc = store.find(SETTINGS, "versionGC");
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNotNull(vgc);
assertEquals(0L, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP));
assertNull(vgc.get(SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP));
@@ -65,7 +66,7 @@ public class VersionGCInitTest {
@Test
public void lazyInitializeWithDetailedGC() throws Exception {
DocumentStore store = ns.getDocumentStore();
- Document vgc = store.find(SETTINGS, "versionGC");
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNull(vgc);
enableDetailGC(ns.getVersionGarbageCollector());
@@ -77,7 +78,7 @@ public class VersionGCInitTest {
store.createOrUpdate(NODES, op);
VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS);
- vgc = store.find(SETTINGS, "versionGC");
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNotNull(vgc);
assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP));
assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP));
@@ -87,13 +88,13 @@ public class VersionGCInitTest {
@Test
public void lazyInitializeWithDetailedGCWithNoData() throws Exception {
DocumentStore store = ns.getDocumentStore();
- Document vgc = store.find(SETTINGS, "versionGC");
+ Document vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNull(vgc);
enableDetailGC(ns.getVersionGarbageCollector());
VersionGCStats stats = ns.getVersionGarbageCollector().gc(1, DAYS);
- vgc = store.find(SETTINGS, "versionGC");
+ vgc = store.find(SETTINGS, SETTINGS_COLLECTION_ID);
assertNotNull(vgc);
assertEquals(stats.oldestModifiedDocTimeStamp, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_TIMESTAMP_PROP));
assertEquals(stats.oldestModifiedDocId, vgc.get(SETTINGS_COLLECTION_DETAILED_GC_DOCUMENT_ID_PROP));