[jackrabbit-oak] 02/02: OAK-9975: Throw exception in consistency check if all repositories do not have marked references to push consistent metrics

[am...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

amitj pushed a commit to branch OAK-9975
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 591c2f91e67f81301fd9f5949ab5b4837e71d5d6
Author: amjain <am...@adobe.com>
AuthorDate: Fri Nov 18 14:48:32 2022 +0530

    OAK-9975: Throw exception in consistency check if all repositories do not have marked references to push consistent metrics
---
 .../oak/plugins/blob/MarkSweepGarbageCollector.java  | 18 ++++++++++++++++++
 .../jackrabbit/oak/plugins/blob/BlobGCTest.java      | 20 ++++++++++++++++++++
 2 files changed, 38 insertions(+)

diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
index 0a172c867e..43b03b52ac 100644
--- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
+++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
@@ -711,6 +711,9 @@ public class MarkSweepGarbageCollector implements BlobGarbageCollector {
 
             // Mark all used blob references
             iterateNodeTree(fs, true);
+            // Move the marked references file to the data store meta area if applicable
+            String uniqueSuffix = UUID.randomUUID().toString();
+            GarbageCollectionType.get(blobStore).addMarked(blobStore, fs, repoId, uniqueSuffix);
             consistencyStatsCollector.updateMarkDuration(sw.elapsed(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
 
             try {
@@ -725,6 +728,21 @@ public class MarkSweepGarbageCollector implements BlobGarbageCollector {
                 // Retrieve all other marked present in the datastore
                 List<DataRecord> refFiles =
                     ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+
+                // Get all the repositories registered
+                List<DataRecord> repoFiles =
+                    ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType());
+                LOG.info("Repositories registered {}", repoFiles);
+
+                // Retrieve repos for which reference files have not been created
+                Set<String> unAvailRepos =
+                    SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles);
+                LOG.info("Repositories with unavailable references {}", unAvailRepos);
+                
+                if (!unAvailRepos.isEmpty()) {
+                    throw new NotAllRepositoryMarkedException("Not all repositories have marked references available");
+                }
+                
                 if (refFiles.size() > 0) {
                     File temp = new File(root, repoId + UUID.randomUUID().toString());
                     copyFile(fs.getMarkedRefs(), temp);
diff --git a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
index a8cb2c822f..41127c2fe9 100644
--- a/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
+++ b/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
@@ -63,6 +63,7 @@ import org.apache.jackrabbit.oak.api.blob.BlobAccessProvider;
 import org.apache.jackrabbit.oak.api.blob.BlobUpload;
 import org.apache.jackrabbit.oak.commons.concurrent.ExecutorCloser;
 import org.apache.jackrabbit.oak.commons.junit.LogCustomizer;
+import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.NotAllRepositoryMarkedException;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
 import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
 import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -81,6 +82,7 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.ExpectedException;
 import org.junit.rules.TemporaryFolder;
 import org.mockito.internal.util.collections.Iterables;
 import org.slf4j.Logger;
@@ -94,6 +96,9 @@ public class BlobGCTest {
 
     @Rule
     public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+    
+    @Rule
+    public ExpectedException expectedEx = ExpectedException.none();
 
     protected Whiteboard wb;
 
@@ -501,6 +506,21 @@ public class BlobGCTest {
         assertStatsBean(globalCollector.getConsistencyOperationStats(), 1, 1, 1);
     }
 
+    @Test
+    public void checkConsistencyFailureNotAllMarked() throws Exception {
+        log.info("Staring checkConsistencyFailureNotAllMarked()");
+        expectedEx.expect(NotAllRepositoryMarkedException.class);
+
+        // Setup a different cluster/repository sharing the blob store
+        MemoryBlobStoreNodeStore secondClusterNodeStore = new MemoryBlobStoreNodeStore(cluster.blobStore);
+        Cluster secondCluster = new Cluster(folder.newFolder(), cluster.blobStore, secondClusterNodeStore, 100);
+        closer.register(secondCluster);
+
+        // Execute mark on the default cluster
+        MarkSweepGarbageCollector globalCollector = cluster.getCollector(0, true, false);
+        globalCollector.checkConsistency();
+    }
+
     @Test
     public void gcCheckDeletedSize() throws Exception {
         log.info("Starting gcCheckDeletedSize()");