You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2018/09/14 10:23:53 UTC
svn commit: r1840904 - in /jackrabbit/oak/trunk/oak-blob-plugins/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/
test/java/org/apache/jackrabbit/oak/plugins/blob/
Author: amitj
Date: Fri Sep 14 10:23:53 2018
New Revision: 1840904
URL: http://svn.apache.org/viewvc?rev=1840904&view=rev
Log:
OAK-7753: Enable collection of stats for DataStore checkConsistency operation
Added statistics for checkConsistency operation
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationsStatsMBean.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java?rev=1840904&r1=1840903&r2=1840904&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java Fri Sep 14 10:23:53 2018
@@ -50,7 +50,7 @@ public interface BlobGarbageCollector {
* @throws Exception
*/
List<GarbageCollectionRepoStats> getStats() throws Exception;
-
+
/**
* Checks for consistency in the blob store and reporting the number of missing blobs.
*
@@ -66,4 +66,12 @@ public interface BlobGarbageCollector {
* @throws Exception
*/
OperationsStatsMBean getOperationStats() throws Exception;
+
+ /**
+ * Returns consistency operation statistics
+ *
+ * @return stats object
+ * @throws Exception
+ */
+ OperationsStatsMBean getConsistencyOperationStats();
}
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1840904&r1=1840903&r2=1840904&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Fri Sep 14 10:23:53 2018
@@ -133,6 +133,11 @@ public class MarkSweepGarbageCollector i
private final OperationStatsCollector statsCollector;
+ /** Operation consistency stats object **/
+ private final GarbageCollectionOperationStats consistencyStats;
+
+ private final OperationStatsCollector consistencyStatsCollector;
+
private boolean traceOutput;
/**
@@ -181,6 +186,9 @@ public class MarkSweepGarbageCollector i
}
this.stats = new GarbageCollectionOperationStats(statisticsProvider);
this.statsCollector = stats.getCollector();
+ this.consistencyStats =
+ new GarbageCollectionOperationStats(statisticsProvider, GarbageCollectionOperationStats.CONSISTENCY_NAME);
+ this.consistencyStatsCollector = consistencyStats.getCollector();
}
public MarkSweepGarbageCollector(
@@ -292,6 +300,11 @@ public class MarkSweepGarbageCollector i
return stats;
}
+ @Override
+ public OperationsStatsMBean getConsistencyOperationStats() {
+ return consistencyStats;
+ }
+
/**
* Mark and sweep. Main entry method for GC.
*
@@ -625,10 +638,13 @@ public class MarkSweepGarbageCollector i
*/
@Override
public long checkConsistency() throws Exception {
+ consistencyStatsCollector.start();
+ Stopwatch sw = Stopwatch.createStarted();
+
boolean threw = true;
GarbageCollectorFileState fs = new GarbageCollectorFileState(root);
long candidates = 0;
-
+
try {
LOG.info("Starting blob consistency check");
@@ -639,7 +655,8 @@ public class MarkSweepGarbageCollector i
// Mark all used blob references
iterateNodeTree(fs, true);
-
+ consistencyStatsCollector.updateMarkDuration(sw.elapsed(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
+
try {
blobIdRetriever.get();
} catch (ExecutionException e) {
@@ -662,11 +679,15 @@ public class MarkSweepGarbageCollector i
if (candidates > 0) {
LOG.warn("Consistency check failure in the the blob store : {}, check missing candidates in file {}",
blobStore, fs.getGcCandidates().getAbsolutePath());
+ consistencyStatsCollector.finishFailure();
+ consistencyStatsCollector.updateNumDeleted(candidates);
}
} finally {
if (!traceOutput && (!LOG.isTraceEnabled() && candidates == 0)) {
Closeables.close(fs, threw);
}
+ sw.stop();
+ consistencyStatsCollector.updateDuration(sw.elapsed(TimeUnit.MILLISECONDS), TimeUnit.MILLISECONDS);
}
return candidates;
}
@@ -998,6 +1019,7 @@ public class MarkSweepGarbageCollector i
class GarbageCollectionOperationStats implements OperationsStatsMBean {
static final String NAME = "DataStoreGarbageCollection";
+ static final String CONSISTENCY_NAME = "DataStoreConsistencyCheck";
static final String START = "COUNTER";
static final String FINISH_FAILURE = "FAILURE";
static final String DURATION = "ACTIVE_TIMER";
@@ -1007,6 +1029,8 @@ public class MarkSweepGarbageCollector i
static final String TOTAL_SIZE_DELETED = "TOTAL_SIZE_DELETED";
static final String NUM_CANDIDATES = "NUM_CANDIDATES";
+ private final String typeName;
+
private CounterStats startCounter;
private CounterStats finishFailureCounter;
private CounterStats numDeletedCounter;
@@ -1017,7 +1041,9 @@ public class MarkSweepGarbageCollector i
private final TimerStats sweepDuration;
private final OperationStatsCollector collector;
- GarbageCollectionOperationStats(StatisticsProvider sp) {
+ GarbageCollectionOperationStats(StatisticsProvider sp, String typeName) {
+ this.typeName = typeName;
+
this.startCounter = sp.getCounterStats(getMetricName(START), StatsOptions.METRICS_ONLY);
this.finishFailureCounter = sp.getCounterStats(getMetricName(FINISH_FAILURE), StatsOptions.METRICS_ONLY);
this.numDeletedCounter = sp.getCounterStats(getMetricName(NUM_BLOBS_DELETED), StatsOptions.METRICS_ONLY);
@@ -1068,6 +1094,11 @@ public class MarkSweepGarbageCollector i
};
}
+
+ GarbageCollectionOperationStats(StatisticsProvider sp) {
+ this(sp, NAME);
+ }
+
private String getMetricName(String name) {
return getName() + "." + name;
}
@@ -1077,7 +1108,7 @@ public class MarkSweepGarbageCollector i
}
@Override public String getName() {
- return TYPE + "." + NAME;
+ return TYPE + "." + typeName;
}
@Override public long getStartCount() {
@@ -1091,5 +1122,17 @@ public class MarkSweepGarbageCollector i
@Override public long duration() {
return duration.getCount();
}
+
+ @Override public long markDuration() {
+ return markDuration.getCount();
+ }
+
+ @Override public long numDeleted() {
+ return numDeletedCounter.getCount();
+ }
+
+ @Override public long sizeDeleted() {
+ return totalSizeDeletedCounter.getCount();
+ }
}
}
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationsStatsMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationsStatsMBean.java?rev=1840904&r1=1840903&r2=1840904&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationsStatsMBean.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/OperationsStatsMBean.java Fri Sep 14 10:23:53 2018
@@ -47,4 +47,22 @@ public interface OperationsStatsMBean {
* @return
*/
long duration();
+
+ /**
+ * Returns the duration of the mark operation
+ * @return
+ */
+ long markDuration();
+
+ /**
+ * Returns the number deleted.
+ * @return
+ */
+ long numDeleted();
+
+ /**
+ * Returns the size deleted.
+ * @return
+ */
+ long sizeDeleted();
}
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java?rev=1840904&r1=1840903&r2=1840904&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCTest.java Fri Sep 14 10:23:53 2018
@@ -82,6 +82,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.commons.codec.binary.Hex.encodeHexString;
+import static org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats.CONSISTENCY_NAME;
import static org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats.FINISH_FAILURE;
import static org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats.NAME;
import static org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector.GarbageCollectionOperationStats.NUM_BLOBS_DELETED;
@@ -216,12 +217,12 @@ public class BlobGCTest {
assertTrue(Sets.symmetricDifference(totalPresent, existingAfterGC).isEmpty());
assertStats(secondCluster.statsProvider, 1, 0, totalAdded.size() - totalPresent.size(),
- totalAdded.size() - totalPresent.size());
+ totalAdded.size() - totalPresent.size(), NAME);
}
@Test
public void noSharedGC() throws Exception {
- log.info("Staring noSharedGC()");
+ log.info("Starting noSharedGC()");
// Setup a different cluster/repository sharing the blob store
MemoryBlobStoreNodeStore secondClusterNodeStore = new MemoryBlobStoreNodeStore(cluster.blobStore);
@@ -234,12 +235,12 @@ public class BlobGCTest {
Set<String> existingAfterGC = executeGarbageCollection(secondCluster, secondCluster.getCollector(0), false);
assertEquals(totalAdded, existingAfterGC);
- assertStats(secondCluster.statsProvider, 1, 0, 0, 0);
+ assertStats(secondCluster.statsProvider, 1, 0, 0, 0, NAME);
}
@Test
public void sharedGCRepositoryCloned() throws Exception {
- log.debug("Running sharedGCRepoCloned()");
+ log.debug("Starting sharedGCRepoCloned()");
// Setup a different cluster/repository sharing the blob store and the repository id
MemoryBlobStoreNodeStore secondClusterNodeStore = new MemoryBlobStoreNodeStore(cluster.blobStore);
@@ -261,18 +262,18 @@ public class BlobGCTest {
@Test
public void gc() throws Exception {
- log.info("Staring gc()");
+ log.info("Starting gc()");
Set<String> existingAfterGC = executeGarbageCollection(cluster, cluster.getCollector(0), false);
assertTrue(Sets.symmetricDifference(cluster.blobStoreState.blobsPresent, existingAfterGC).isEmpty());
assertStats(cluster.statsProvider, 1, 0,
cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size(),
- cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size());
+ cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size(), NAME);
}
@Test
public void noGc() throws Exception {
- log.info("Staring noGc()");
+ log.info("Starting noGc()");
long afterSetupTime = clock.getTime();
log.info("after setup time {}", afterSetupTime);
@@ -282,12 +283,45 @@ public class BlobGCTest {
false);
assertTrue(Sets.symmetricDifference(cluster.blobStoreState.blobsAdded, existingAfterGC).isEmpty());
assertStats(cluster.statsProvider, 1, 0, 0,
- cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size());
+ cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size(), NAME);
+ }
+
+ @Test
+ public void checkConsistency() throws Exception {
+ log.info("Starting checkConsistency()");
+
+ long afterSetupTime = clock.getTime();
+ log.info("after setup time {}", afterSetupTime);
+
+ MarkSweepGarbageCollector collector = cluster.getCollector(0);
+ long missing = collector.checkConsistency();
+
+ assertEquals(0, missing);
+ assertStats(cluster.statsProvider, 1, 0, 0, 0, CONSISTENCY_NAME);
+ assertStatsBean(collector.getConsistencyOperationStats(), 1, 0, 0);
+ }
+
+ @Test
+ public void checkConsistencyFailure() throws Exception {
+ log.info("Starting checkConsistencyFailure()");
+
+ long afterSetupTime = clock.getTime();
+ log.info("after setup time {}", afterSetupTime);
+
+ cluster.blobStore
+ .countDeleteChunks(Lists.newArrayList(Iterators.getLast(cluster.blobStoreState.blobsPresent.iterator())),
+ 0);
+ MarkSweepGarbageCollector collector = cluster.getCollector(0);
+ long missing = collector.checkConsistency();
+
+ assertEquals(1, missing);
+ assertStats(cluster.statsProvider, 1, 1, 1, 0, CONSISTENCY_NAME);
+ assertStatsBean(collector.getConsistencyOperationStats(), 1, 1, 1);
}
@Test
public void gcCheckDeletedSize() throws Exception {
- log.info("Staring gcCheckDeletedSize()");
+ log.info("Starting gcCheckDeletedSize()");
// Capture logs for the second round of gc
LogCustomizer customLogs = LogCustomizer
@@ -305,21 +339,20 @@ public class BlobGCTest {
assertTrue(customLogs.getLogs().get(0).contains(String.valueOf(deletedSize)));
assertStats(cluster.statsProvider, 1, 0,
cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size(),
- cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size());
- assertEquals(deletedSize, getStatCount(cluster.statsProvider, TOTAL_SIZE_DELETED));
-
+ cluster.blobStoreState.blobsAdded.size() - cluster.blobStoreState.blobsPresent.size(), NAME);
+ assertEquals(deletedSize, getStatCount(cluster.statsProvider, NAME, TOTAL_SIZE_DELETED));
customLogs.finished();
assertTrue(Sets.symmetricDifference(cluster.blobStoreState.blobsPresent, existingAfterGC).isEmpty());
}
@Test
public void gcMarkOnly() throws Exception {
- log.info("Staring gcMarkOnly()");
+ log.info("Starting gcMarkOnly()");
Set<String> existingAfterGC =
executeGarbageCollection(cluster, cluster.getCollector(0),true);
assertTrue(Sets.symmetricDifference(cluster.blobStoreState.blobsAdded, existingAfterGC).isEmpty());
- assertStats(cluster.statsProvider, 1, 0, 0, 0);
+ assertStats(cluster.statsProvider, 1, 0, 0, 0, NAME);
}
protected Set<String> executeGarbageCollection(Cluster cluster, MarkSweepGarbageCollector collector, boolean markOnly)
@@ -333,16 +366,25 @@ public class BlobGCTest {
return existingAfterGC;
}
- private void assertStats(StatisticsProvider statsProvider, int start, int failure, long deleted, long candidates) {
- assertEquals("Start counter mismatch", start, getStatCount(statsProvider, START));
- assertEquals("Finish error mismatch", failure, getStatCount(statsProvider, FINISH_FAILURE));
- assertEquals("Num deleted mismatch", deleted, getStatCount(statsProvider, NUM_BLOBS_DELETED));
- assertEquals("Num candidates mismatch", candidates, getStatCount(statsProvider, NUM_CANDIDATES));
+ private void assertStats(StatisticsProvider statsProvider, int start, int failure, long deleted, long candidates,
+ String typeName) {
+
+ assertEquals("Start counter mismatch", start, getStatCount(statsProvider, typeName, START));
+ assertEquals("Finish error mismatch", failure, getStatCount(statsProvider, typeName, FINISH_FAILURE));
+ assertEquals("Num deleted mismatch", deleted, getStatCount(statsProvider, typeName, NUM_BLOBS_DELETED));
+ assertEquals("Num candidates mismatch", candidates, getStatCount(statsProvider, typeName, NUM_CANDIDATES));
+ }
+
+
+ private void assertStatsBean(OperationsStatsMBean mbean, int start, int failure, long deleted) {
+ assertEquals("Start counter mismatch", start, mbean.getStartCount());
+ assertEquals("Finish error mismatch", failure, mbean.getFailureCount());
+ assertEquals("Num deleted mismatch", deleted, mbean.numDeleted());
}
- private long getStatCount(StatisticsProvider statsProvider, String name) {
+ private long getStatCount(StatisticsProvider statsProvider, String typeName, String name) {
return statsProvider.getCounterStats(
- TYPE + "." + NAME + "." + name, METRICS_ONLY).getCount();
+ TYPE + "." + typeName + "." + name, METRICS_ONLY).getCount();
}
protected Set<String> iterate(GarbageCollectableBlobStore blobStore) throws Exception {