You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2015/09/11 06:42:11 UTC
svn commit: r1702374 - in /jackrabbit/oak/branches/1.2: ./
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segmen...
Author: amitj
Date: Fri Sep 11 04:42:09 2015
New Revision: 1702374
URL: http://svn.apache.org/r1702374
Log:
OAK-3031: [Blob GC] Mbean for reporting shared repository GC stats
Merged revision 1700403 from trunk
Added:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectionRepoStats.java
- copied unchanged from r1700403, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectionRepoStats.java
Modified:
jackrabbit/oak/branches/1.2/ (props changed)
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
Propchange: jackrabbit/oak/branches/1.2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Sep 11 04:42:09 2015
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820
,1684868,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521,1695540,1696194,1696242,1696285,1696578,1696759,1696916,1697363,1697373,1697410,1697
582,1697589,1697616,1697672,1700191,1700231,1700397,1700506,1700571,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701948,1701955,1701959,1701986,1702022,1702272
+/jackrabbit/oak/trunk:1672350,1672468,1672537,1672603,1672642,1672644,1672834-1672835,1673351,1673410,1673414-1673415,1673436,1673644,1673662-1673664,1673669,1673695,1673713,1673738,1673787,1673791,1674046,1674065,1674075,1674107,1674228,1674780,1674880,1675054-1675055,1675319,1675332,1675354,1675357,1675382,1675555,1675566,1675593,1676198,1676237,1676407,1676458,1676539,1676670,1676693,1676703,1676725,1677579,1677581,1677609,1677611,1677774,1677788,1677797,1677804,1677806,1677939,1677991,1678023,1678095-1678096,1678124,1678171,1678173,1678211,1678323,1678758,1678938,1678954,1679144,1679165,1679191,1679232,1679235,1679503,1679958,1679961,1680170,1680172,1680182,1680222,1680232,1680236,1680461,1680633,1680643,1680747,1680805-1680806,1680903,1681282,1681767,1681918,1681955,1682042,1682218,1682235,1682437,1682494,1682555,1682855,1682904,1683059,1683089,1683213,1683249,1683259,1683278,1683323,1683687,1683700,1684174-1684175,1684186,1684376,1684442,1684561,1684570,1684601,1684618,1684820
,1684868,1685023,1685075,1685370,1685552,1685589-1685590,1685840,1685964,1685977,1685989,1685999,1686023,1686032,1686097,1686162,1686229,1686234,1686253,1686414,1686780,1686854,1686857,1686971,1687053-1687055,1687175,1687196,1687198,1687220,1687239-1687240,1687301,1687441,1687553,1688089-1688090,1688172,1688179,1688349,1688421,1688436,1688453,1688616,1688622,1688634,1688636,1688817,1689003-1689004,1689008,1689577,1689581,1689623,1689810,1689828,1689831,1689833,1689903,1690017,1690043,1690047,1690057,1690247,1690249,1690634-1690637,1690650,1690669,1690674,1690885,1690941,1691139,1691151,1691159,1691167,1691183,1691188,1691210,1691280,1691307,1691331-1691333,1691345,1691384-1691385,1691401,1691509,1692133-1692134,1692156,1692250,1692274,1692363,1692382,1692478,1692955,1693002,1693030,1693209,1693421,1693525-1693526,1694007,1694393-1694394,1695050,1695122,1695280,1695299,1695457,1695482,1695507,1695521,1695540,1696194,1696242,1696285,1696578,1696759,1696916,1697363,1697373,1697410,1697
582,1697589,1697616,1697672,1700191,1700231,1700397,1700403,1700506,1700571,1700727,1700749,1700769,1700775,1701065,1701619,1701733,1701743,1701750,1701768,1701806,1701810,1701814,1701948,1701955,1701959,1701986,1702022,1702272
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java Fri Sep 11 04:42:09 2015
@@ -21,16 +21,28 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Preconditions.checkNotNull;
import static java.lang.System.nanoTime;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.oak.management.ManagementOperation.Status.formatTime;
import static org.apache.jackrabbit.oak.management.ManagementOperation.done;
import static org.apache.jackrabbit.oak.management.ManagementOperation.newManagementOperation;
+import java.util.Date;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.Executor;
import javax.annotation.Nonnull;
import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.CompositeDataSupport;
+import javax.management.openmbean.CompositeType;
+import javax.management.openmbean.OpenDataException;
+import javax.management.openmbean.OpenType;
+import javax.management.openmbean.SimpleType;
+import javax.management.openmbean.TabularData;
+import javax.management.openmbean.TabularDataSupport;
+import javax.management.openmbean.TabularType;
+import org.apache.jackrabbit.oak.commons.jmx.AnnotatedStandardMBean;
import org.apache.jackrabbit.oak.management.ManagementOperation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,7 +50,7 @@ import org.slf4j.LoggerFactory;
/**
* Default implementation of {@link BlobGCMBean} based on a {@link BlobGarbageCollector}.
*/
-public class BlobGC implements BlobGCMBean {
+public class BlobGC extends AnnotatedStandardMBean implements BlobGCMBean {
private static final Logger log = LoggerFactory.getLogger(BlobGC.class);
public static final String OP_NAME = "Blob garbage collection";
@@ -55,6 +67,7 @@ public class BlobGC implements BlobGCMBe
public BlobGC(
@Nonnull BlobGarbageCollector blobGarbageCollector,
@Nonnull Executor executor) {
+ super(BlobGCMBean.class);
this.blobGarbageCollector = checkNotNull(blobGarbageCollector);
this.executor = checkNotNull(executor);
}
@@ -81,4 +94,73 @@ public class BlobGC implements BlobGCMBe
public CompositeData getBlobGCStatus() {
return gcOp.getStatus().toCompositeData();
}
+
+ @Override
+ public TabularData getGlobalMarkStats() {
+ TabularDataSupport tds;
+ try {
+ TabularType tt = new TabularType(BlobGC.class.getName(),
+ "Garbage collection global mark phase Stats",
+ TYPE,
+ new String[] {"repositoryId"});
+ tds = new TabularDataSupport(tt);
+ List<GarbageCollectionRepoStats> stats = blobGarbageCollector.getStats();
+ for (GarbageCollectionRepoStats stat : stats) {
+ tds.put(toCompositeData(stat));
+ }
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ return tds;
+ }
+
+ private CompositeDataSupport toCompositeData(GarbageCollectionRepoStats statObj) throws OpenDataException {
+ Object[] values = new Object[] {
+ statObj.getRepositoryId(),
+ (statObj.getLastModified() == 0 ? "" : (new Date(statObj.getLastModified()))).toString(),
+ statObj.getLength(),
+ humanReadableByteCount(statObj.getLength()),
+ statObj.getNumLines()
+ };
+ return new CompositeDataSupport(TYPE, FIELD_NAMES, values);
+ }
+
+ private static final String[] FIELD_NAMES = new String[] {
+ "repositoryId",
+ "referencesLastModifiedTime",
+ "referenceFileSizeBytes",
+ "referencesFileSize",
+ "numReferences",
+ };
+
+ private static final String[] FIELD_DESCRIPTIONS = new String[] {
+ "Repository ID",
+ "Last modified time of references",
+ "References file size in bytes",
+ "References file size in human readable format",
+ "Number of references"
+ };
+
+ private static final OpenType[] FIELD_TYPES = new OpenType[] {
+ SimpleType.STRING,
+ SimpleType.STRING,
+ SimpleType.LONG,
+ SimpleType.STRING,
+ SimpleType.INTEGER
+ };
+
+ private static final CompositeType TYPE = createCompositeType();
+
+ private static CompositeType createCompositeType() {
+ try {
+ return new CompositeType(
+ GarbageCollectionRepoStats.class.getName(),
+ "Composite data type for datastore GC statistics",
+ FIELD_NAMES,
+ FIELD_DESCRIPTIONS,
+ FIELD_TYPES);
+ } catch (OpenDataException e) {
+ throw new IllegalStateException(e);
+ }
+ }
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java Fri Sep 11 04:42:09 2015
@@ -19,8 +19,12 @@
package org.apache.jackrabbit.oak.plugins.blob;
+import org.apache.jackrabbit.oak.commons.jmx.Description;
+import org.apache.jackrabbit.oak.commons.jmx.Name;
+
import javax.annotation.Nonnull;
import javax.management.openmbean.CompositeData;
+import javax.management.openmbean.TabularData;
/**
* MBean for starting and monitoring the progress of
@@ -37,8 +41,11 @@ public interface BlobGCMBean {
* @param markOnly whether to only mark references and not sweep in the mark and sweep operation.
* @return the status of the operation right after it was initiated
*/
- CompositeData startBlobGC(boolean markOnly);
-
+ CompositeData startBlobGC(@Name("markOnly")
+ @Description("Set to true to only mark references and not sweep in the mark and sweep operation. " +
+ "This mode is to be used when the underlying BlobStore is shared between multiple " +
+ "different repositories. For all other cases set it to false to perform full garbage collection")
+ boolean markOnly);
/**
* Data store garbage collection status
*
@@ -47,5 +54,12 @@ public interface BlobGCMBean {
*/
@Nonnull
CompositeData getBlobGCStatus();
+
+ /**
+ * Show details of the data Store garbage collection process.
+ *
+ * @return List of available repositories and their status
+ */
+ TabularData getGlobalMarkStats();
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java Fri Sep 11 04:42:09 2015
@@ -16,6 +16,8 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import java.util.List;
+
/**
* Interface for blob garbage collector
*/
@@ -29,4 +31,12 @@ public interface BlobGarbageCollector {
* @throws Exception the exception
*/
void collectGarbage(boolean markOnly) throws Exception;
+
+ /**
+ * Retuns the list of stats
+ *
+ * @return stats
+ * @throws Exception
+ */
+ List<GarbageCollectionRepoStats> getStats() throws Exception;
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Fri Sep 11 04:42:09 2015
@@ -23,9 +23,12 @@ import java.io.FileInputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.LineNumberReader;
import java.sql.Timestamp;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
@@ -34,12 +37,14 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.base.Charsets;
+import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.StandardSystemProperty;
import com.google.common.base.Stopwatch;
import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
import com.google.common.collect.PeekingIterator;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
@@ -57,6 +62,8 @@ import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import static com.google.common.collect.Lists.newArrayList;
+
/**
* Mark and sweep garbage collector.
*
@@ -74,8 +81,6 @@ public class MarkSweepGarbageCollector i
public static final int DEFAULT_BATCH_COUNT = 2048;
- public static enum State {NOT_RUNNING, MARKING, SWEEPING}
-
/** The last modified time before current time of blobs to consider for garbage collection. */
private final long maxLastModifiedInterval;
@@ -84,19 +89,15 @@ public class MarkSweepGarbageCollector i
/** Helper class to mark blob references which **/
private final BlobReferenceRetriever marker;
-
- /** The garbage collector file state */
- private final GarbageCollectorFileState fs;
private final Executor executor;
/** The batch count. */
private final int batchCount;
- private String repoId;
+ private final String repoId;
- /** Flag to indicate the state of the gc **/
- private State state = State.NOT_RUNNING;
+ private final String root;
/**
* Creates an instance of MarkSweepGarbageCollector
@@ -127,7 +128,7 @@ public class MarkSweepGarbageCollector i
this.batchCount = batchCount;
this.maxLastModifiedInterval = maxLastModifiedInterval;
this.repoId = repositoryId;
- fs = new GarbageCollectorFileState(root);
+ this.root = root;
}
/**
@@ -158,14 +159,55 @@ public class MarkSweepGarbageCollector i
}
/**
- * Gets the state of the gc process.
- *
- * @return the state
+ * Returns the stats related to GC for all repos
+ *
+ * @return a list of GarbageCollectionRepoStats objects
+ * @throws Exception
*/
- public State getState() {
- return state;
+ @Override
+ public List<GarbageCollectionRepoStats> getStats() throws Exception {
+ List<GarbageCollectionRepoStats> stats = newArrayList();
+ if (SharedDataStoreUtils.isShared(blobStore)) {
+ // Get all the references available
+ List<DataRecord> refFiles =
+ ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REFERENCES.getType());
+ Map<String, DataRecord> references = Maps.uniqueIndex(refFiles, new Function<DataRecord, String>() {
+ @Override
+ public String apply(DataRecord input) {
+ return SharedStoreRecordType.REFERENCES.getIdFromName(input.getIdentifier().toString());
+ }
+ });
+
+ // Get all the repositories registered
+ List<DataRecord> repoFiles =
+ ((SharedDataStore) blobStore).getAllMetadataRecords(SharedStoreRecordType.REPOSITORY.getType());
+
+ for (DataRecord repoRec : repoFiles) {
+ String repoId = SharedStoreRecordType.REFERENCES.getIdFromName(repoRec.getIdentifier().toString());
+ GarbageCollectionRepoStats stat = new GarbageCollectionRepoStats();
+ stat.setRepositoryId(repoId);
+ if (references.containsKey(repoId)) {
+ DataRecord refRec = references.get(repoId);
+ stat.setLastModified(refRec.getLastModified());
+ stat.setLength(refRec.getLength());
+
+ LineNumberReader reader = null;
+ try {
+ reader = new LineNumberReader(new InputStreamReader(refRec.getStream()));
+ while (reader.readLine() != null) {
+ }
+ stat.setNumLines(reader.getLineNumber());
+ } finally {
+ Closeables.close(reader, true);
+ }
+ }
+ stats.add(stat);
+ }
+ }
+ return stats;
}
+
/**
* Mark and sweep. Main entry method for GC.
*
@@ -174,13 +216,14 @@ public class MarkSweepGarbageCollector i
*/
private void markAndSweep(boolean markOnly) throws Exception {
boolean threw = true;
+ GarbageCollectorFileState fs = new GarbageCollectorFileState(root);
try {
Stopwatch sw = Stopwatch.createStarted();
LOG.info("Starting Blob garbage collection");
- mark();
+ mark(fs);
if (!markOnly) {
- long deleteCount = sweep();
+ long deleteCount = sweep(fs);
threw = false;
LOG.info("Blob garbage collection completed in {}. Number of blobs deleted [{}]", sw.toString(),
@@ -190,19 +233,18 @@ public class MarkSweepGarbageCollector i
if (!LOG.isTraceEnabled()) {
Closeables.close(fs, threw);
}
- state = State.NOT_RUNNING;
}
}
/**
* Mark phase of the GC.
+ * @param fs the garbage collector file state
*/
- private void mark() throws IOException, DataStoreException {
- state = State.MARKING;
+ private void mark(GarbageCollectorFileState fs) throws IOException, DataStoreException {
LOG.debug("Starting mark phase of the garbage collector");
// Mark all used references
- iterateNodeTree();
+ iterateNodeTree(fs);
// Move the marked references file to the data store meta area if applicable
GarbageCollectionType.get(blobStore).addMarked(blobStore, fs, repoId);
@@ -213,10 +255,11 @@ public class MarkSweepGarbageCollector i
/**
* Difference phase where the GC candidates are identified.
*
+ * @param fs the garbage collector file state
* @throws IOException
* Signals that an I/O exception has occurred.
*/
- private void difference() throws IOException {
+ private void difference(GarbageCollectorFileState fs) throws IOException {
LOG.debug("Starting difference phase of the garbage collector");
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
@@ -226,7 +269,7 @@ public class MarkSweepGarbageCollector i
BufferedWriter bufferWriter = null;
try {
bufferWriter = Files.newWriter(fs.getGcCandidates(), Charsets.UTF_8);
- List<String> expiredSet = Lists.newArrayList();
+ List<String> expiredSet = newArrayList();
int numCandidates = 0;
while (iter.hasNext()) {
@@ -279,8 +322,9 @@ public class MarkSweepGarbageCollector i
*
* @return the number of blobs deleted
* @throws Exception the exception
+ * @param fs the garbage collector file state
*/
- private long sweep() throws Exception {
+ private long sweep(GarbageCollectorFileState fs) throws Exception {
long earliestRefAvailTime;
// Merge all the blob references available from all the reference files in the data store meta store
// Only go ahead if merge succeeded
@@ -293,14 +337,13 @@ public class MarkSweepGarbageCollector i
}
// Find all blob references after iterating over the whole repository
- (new BlobIdRetriever()).call();
+ (new BlobIdRetriever(fs)).call();
// Calculate the references not used
- difference();
+ difference(fs);
long count = 0;
long deleted = 0;
- state = State.SWEEPING;
LOG.debug("Starting sweep phase of the garbage collector");
LOG.debug("Sweeping blobs with modified time > than the configured max deleted time ({}). " +
timestampToString(getLastMaxModifiedTime(earliestRefAvailTime)));
@@ -309,7 +352,7 @@ public class MarkSweepGarbageCollector i
LineIterator iterator =
FileUtils.lineIterator(fs.getGcCandidates(), Charsets.UTF_8.name());
- List<String> ids = Lists.newArrayList();
+ List<String> ids = newArrayList();
while (iterator.hasNext()) {
ids.add(iterator.next());
@@ -317,7 +360,7 @@ public class MarkSweepGarbageCollector i
if (ids.size() >= getBatchCount()) {
count += ids.size();
deleted += sweepInternal(ids, exceptionQueue, earliestRefAvailTime);
- ids = Lists.newArrayList();
+ ids = newArrayList();
}
}
if (!ids.isEmpty()) {
@@ -329,7 +372,7 @@ public class MarkSweepGarbageCollector i
try {
if (!exceptionQueue.isEmpty()) {
writer = Files.newWriter(fs.getGarbage(), Charsets.UTF_8);
- saveBatchToFile(Lists.newArrayList(exceptionQueue), writer);
+ saveBatchToFile(newArrayList(exceptionQueue), writer);
}
} finally {
LineIterator.closeQuietly(iterator);
@@ -400,8 +443,9 @@ public class MarkSweepGarbageCollector i
/**
* Iterates the complete node tree and collect all blob references
+ * @param fs the garbage collector file state
*/
- private void iterateNodeTree() throws IOException {
+ private void iterateNodeTree(GarbageCollectorFileState fs) throws IOException {
final BufferedWriter writer = Files.newWriter(fs.getMarkedRefs(), Charsets.UTF_8);
final AtomicInteger count = new AtomicInteger();
try {
@@ -458,6 +502,12 @@ public class MarkSweepGarbageCollector i
* BlobIdRetriever class to retrieve all blob ids.
*/
private class BlobIdRetriever implements Callable<Integer> {
+ private final GarbageCollectorFileState fs;
+
+ public BlobIdRetriever(GarbageCollectorFileState fs) {
+ this.fs = fs;
+ }
+
@Override
public Integer call() throws Exception {
LOG.debug("Starting retrieve of all blobs");
@@ -467,7 +517,7 @@ public class MarkSweepGarbageCollector i
bufferWriter = new BufferedWriter(
new FileWriter(fs.getAvailableRefs()));
Iterator<String> idsIter = blobStore.getAllChunkIds(0);
- List<String> ids = Lists.newArrayList();
+ List<String> ids = newArrayList();
while (idsIter.hasNext()) {
ids.add(idsIter.next());
@@ -614,7 +664,7 @@ public class MarkSweepGarbageCollector i
SharedDataStoreUtils.refsNotAvailableFromRepos(repoFiles, refFiles);
if (unAvailRepos.isEmpty()) {
// List of files to be merged
- List<File> files = Lists.newArrayList();
+ List<File> files = newArrayList();
for (DataRecord refFile : refFiles) {
File file = GarbageCollectorFileState.copy(refFile.getStream());
files.add(file);
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java Fri Sep 11 04:42:09 2015
@@ -578,14 +578,8 @@ public class DocumentNodeStoreService {
final long blobGcMaxAgeInSecs = toLong(prop(PROP_BLOB_GC_MAX_AGE), DEFAULT_BLOB_GC_MAX_AGE);
if (store.getBlobStore() instanceof GarbageCollectableBlobStore) {
- BlobGarbageCollector gc = new BlobGarbageCollector() {
- @Override
- public void collectGarbage(boolean sweep) throws Exception {
- store.createBlobGarbageCollector(blobGcMaxAgeInSecs,
- ClusterRepositoryInfo.getId(mk.getNodeStore()))
- .collectGarbage(sweep);
- }
- };
+ BlobGarbageCollector gc = store.createBlobGarbageCollector(blobGcMaxAgeInSecs,
+ ClusterRepositoryInfo.getId(mk.getNodeStore()));
registrations.add(registerMBean(whiteboard, BlobGCMBean.class, new BlobGC(gc, executor),
BlobGCMBean.TYPE, "Document node store blob garbage collection"));
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java Fri Sep 11 04:42:09 2015
@@ -378,17 +378,11 @@ public class SegmentNodeStoreService ext
}
if (store.getBlobStore() instanceof GarbageCollectableBlobStore) {
- BlobGarbageCollector gc = new BlobGarbageCollector() {
- @Override
- public void collectGarbage(boolean sweep) throws Exception {
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
- (GarbageCollectableBlobStore) store.getBlobStore(),
- executor, TimeUnit.SECONDS.toMillis(blobGcMaxAgeInSecs),
- ClusterRepositoryInfo.getId(delegate));
- gc.collectGarbage(sweep);
- }
- };
+ BlobGarbageCollector gc = new MarkSweepGarbageCollector(
+ new SegmentBlobReferenceRetriever(store.getTracker()),
+ (GarbageCollectableBlobStore) store.getBlobStore(),
+ executor, TimeUnit.SECONDS.toMillis(blobGcMaxAgeInSecs),
+ ClusterRepositoryInfo.getId(delegate));
blobGCRegistration = registerMBean(whiteboard, BlobGCMBean.class, new BlobGC(gc, executor),
BlobGCMBean.TYPE, "Segment node store blob garbage collection");
Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1702374&r1=1702373&r2=1702374&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Fri Sep 11 04:42:09 2015
@@ -41,6 +41,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.blob.BlobGarbageCollector;
+import org.apache.jackrabbit.oak.plugins.blob.GarbageCollectionRepoStats;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
import org.apache.jackrabbit.oak.plugins.blob.SharedDataStore;
import org.apache.jackrabbit.oak.plugins.blob.datastore.DataStoreBlobStore;
@@ -134,6 +135,33 @@ public class SharedBlobStoreGCTest {
}
@Test
+ public void testGCStats() throws Exception {
+ log.debug("Running testGCStats()");
+ // Only run the mark phase on both the clusters to get the stats
+ cluster1.gc.collectGarbage(true);
+ cluster2.gc.collectGarbage(true);
+
+ Set<String> actualRepoIds = Sets.newHashSet();
+ actualRepoIds.add(cluster1.repoId);
+ actualRepoIds.add(cluster2.repoId);
+
+ Set<Integer> actualNumBlobs = Sets.newHashSet();
+ actualNumBlobs.add(cluster1.initBlobs.size());
+ actualNumBlobs.add(cluster2.initBlobs.size());
+
+ List<GarbageCollectionRepoStats> statsList = cluster1.gc.getStats();
+ Set<Integer> observedNumBlobs = Sets.newHashSet();
+ Set<String> observedRepoIds = Sets.newHashSet();
+ for (GarbageCollectionRepoStats stat : statsList) {
+ observedNumBlobs.add(stat.getNumLines());
+ observedRepoIds.add(stat.getRepositoryId());
+ }
+
+ Assert.assertTrue(Sets.difference(actualNumBlobs, observedNumBlobs).isEmpty());
+ Assert.assertTrue(Sets.difference(actualRepoIds, observedRepoIds).isEmpty());
+ }
+
+ @Test
// GC should fail
public void testOnly1ClusterMark() throws Exception {
log.debug("Running testOnly1ClusterMark()");
@@ -182,7 +210,7 @@ public class SharedBlobStoreGCTest {
private int seed;
private BlobGarbageCollector gc;
private Date startDate;
-
+ private String repoId;
private Set<String> initBlobs = new HashSet<String>();
protected Set<String> getInitBlobs() {
@@ -192,19 +220,14 @@ public class SharedBlobStoreGCTest {
public Cluster(final DocumentNodeStore ds, final String repoId, int seed)
throws IOException {
this.ds = ds;
- this.gc = new BlobGarbageCollector() {
- @Override
- public void collectGarbage(boolean markOnly) throws Exception {
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
+ this.gc = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(ds),
(GarbageCollectableBlobStore) ds.getBlobStore(),
MoreExecutors.sameThreadExecutor(),
"./target", 5, 0, repoId);
- gc.collectGarbage(markOnly);
- }
- };
this.startDate = new Date();
this.seed = seed;
+ this.repoId = repoId;
}
/**