You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2015/09/11 06:13:15 UTC
svn commit: r1702371 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/
main/java/org/apache/jackrabbit/oak/plugins/document/
main/java/org/apache/jackrabbit/oak/plugins/document/mongo/
main/java/org/apache/jackr...
Author: amitj
Date: Fri Sep 11 04:13:15 2015
New Revision: 1702371
URL: http://svn.apache.org/r1702371
Log:
OAK-3184: Consistency checker for data/blob store
Reports the number of missing blobs if any and logs the blobId and the node path from where referenced.
Added:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java (with props)
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobIT.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGC.java Fri Sep 11 04:13:15 2015
@@ -59,7 +59,11 @@ public class BlobGC extends AnnotatedSt
private final Executor executor;
private ManagementOperation<String> gcOp = done(OP_NAME, "");
-
+
+ public static final String CONSISTENCY_OP_NAME = "Blob consistency check";
+
+ private ManagementOperation<String> consistencyOp = done(CONSISTENCY_OP_NAME, "");
+
/**
* @param blobGarbageCollector Blob garbage collector
* @param executor executor for running the garbage collection task
@@ -114,6 +118,23 @@ public class BlobGC extends AnnotatedSt
return tds;
}
+ @Override
+ public CompositeData checkConsistency() {
+ if (consistencyOp.isDone()) {
+ consistencyOp = newManagementOperation(CONSISTENCY_OP_NAME, new Callable<String>() {
+ @Override
+ public String call() throws Exception {
+ long t0 = nanoTime();
+ long missing = blobGarbageCollector.checkConsistency();
+ return missing + "missing blobs found (details in the log). Consistency check completed in "
+ + formatTime(nanoTime() - t0);
+ }
+ });
+ executor.execute(consistencyOp);
+ }
+ return consistencyOp.getStatus().toCompositeData();
+ }
+
private CompositeDataSupport toCompositeData(GarbageCollectionRepoStats statObj) throws OpenDataException {
Object[] values = new Object[] {
statObj.getRepositoryId(),
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGCMBean.java Fri Sep 11 04:13:15 2015
@@ -61,5 +61,11 @@ public interface BlobGCMBean {
* @return List of available repositories and their status
*/
TabularData getGlobalMarkStats();
-
+
+ /**
+ * Data Store consistency check
+ *
+ * @return the missing blobs
+ */
+ CompositeData checkConsistency();
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/BlobGarbageCollector.java Fri Sep 11 04:13:15 2015
@@ -39,4 +39,12 @@ public interface BlobGarbageCollector {
* @throws Exception
*/
List<GarbageCollectionRepoStats> getStats() throws Exception;
+
+ /**
+ * Checks for consistency in the blob store and reporting the number of missing blobs.
+ *
+ * @return number of inconsistencies
+ * @throws Exception
+ */
+ long checkConsistency() throws Exception;
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/GarbageCollectorFileState.java Fri Sep 11 04:13:15 2015
@@ -134,6 +134,20 @@ class GarbageCollectorFileState implemen
merge(ExternalSort.sortInBatch(file, lexComparator, true), sorted);
Files.move(sorted, file);
}
+
+ /**
+ * Sorts the given file externally with the given comparator.
+ *
+ * @param file file whose contents needs to be sorted
+ * @param comparator to compare
+ * @throws IOException
+ */
+ public static void sort(File file, Comparator<String> comparator) throws IOException {
+ File sorted = createTempFile();
+ merge(ExternalSort.sortInBatch(file, comparator, true), sorted);
+ Files.move(sorted, file);
+ }
+
public static void merge(List<File> files, File output) throws IOException {
ExternalSort.mergeSortedFiles(
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Fri Sep 11 04:13:15 2015
@@ -26,12 +26,14 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.sql.Timestamp;
+import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -49,6 +51,7 @@ import com.google.common.collect.Peeking
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import com.google.common.util.concurrent.ListenableFutureTask;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;
import org.apache.jackrabbit.core.data.DataRecord;
@@ -80,7 +83,9 @@ public class MarkSweepGarbageCollector i
public static final String TEMP_DIR = StandardSystemProperty.JAVA_IO_TMPDIR.value();
public static final int DEFAULT_BATCH_COUNT = 2048;
-
+
+ public static final String DELIM = ",";
+
/** The last modified time before current time of blobs to consider for garbage collection. */
private final long maxLastModifiedInterval;
@@ -265,13 +270,18 @@ public class MarkSweepGarbageCollector i
FileLineDifferenceIterator iter = new FileLineDifferenceIterator(
fs.getMarkedRefs(),
fs.getAvailableRefs());
+ calculateDifference(fs, iter);
+ LOG.debug("Ending difference phase of the garbage collector");
+ }
+
+ private long calculateDifference(GarbageCollectorFileState fs, FileLineDifferenceIterator iter) throws IOException {
+ long numCandidates = 0;
BufferedWriter bufferWriter = null;
try {
bufferWriter = Files.newWriter(fs.getGcCandidates(), Charsets.UTF_8);
List<String> expiredSet = newArrayList();
- int numCandidates = 0;
while (iter.hasNext()) {
expiredSet.add(iter.next());
if (expiredSet.size() > getBatchCount()) {
@@ -284,15 +294,14 @@ public class MarkSweepGarbageCollector i
numCandidates += expiredSet.size();
saveBatchToFile(expiredSet, bufferWriter);
}
- LOG.debug("Found GC candidates - " + numCandidates);
+ LOG.debug("Found candidates - " + numCandidates);
} finally {
IOUtils.closeQuietly(bufferWriter);
IOUtils.closeQuietly(iter);
}
-
- LOG.debug("Ending difference phase of the garbage collector");
+ return numCandidates;
}
-
+
/**
* Sweep phase of gc candidate deletion.
* <p>
@@ -456,16 +465,18 @@ public class MarkSweepGarbageCollector i
private final boolean debugMode = LOG.isTraceEnabled();
@Override
- public void addReference(String blobId) {
+ public void addReference(String blobId, String nodeId) {
if (debugMode) {
- LOG.trace("BlobId : {}", blobId);
+ LOG.trace("BlobId : {}, NodeId : {}", blobId, nodeId);
}
try {
Iterator<String> idIter = blobStore.resolveChunks(blobId);
+ Joiner delimJoiner = Joiner.on(DELIM).skipNulls();
while (idIter.hasNext()) {
String id = idIter.next();
- idBatch.add(id);
+
+ idBatch.add(delimJoiner.join(id, nodeId));
if (idBatch.size() >= getBatchCount()) {
saveBatchToFile(idBatch, writer);
@@ -490,14 +501,66 @@ public class MarkSweepGarbageCollector i
);
LOG.info("Number of valid blob references marked under mark phase of " +
"Blob garbage collection [{}]", count.get());
- // sort the marked references
- GarbageCollectorFileState.sort(fs.getMarkedRefs());
+ // sort the marked references with the first part of the key
+ GarbageCollectorFileState.sort(fs.getMarkedRefs(),
+ new Comparator<String>() {
+ @Override
+ public int compare(String s1, String s2) {
+ return s1.split(DELIM)[0].compareTo(s2.split(DELIM)[0]);
+ }
+ });
} finally {
IOUtils.closeQuietly(writer);
}
}
-
-
+
+ /**
+ * Checks for the DataStore consistency and reports the number of missing blobs still referenced.
+ *
+ * @return the missing blobs
+ * @throws Exception
+ */
+ @Override
+ public long checkConsistency() throws Exception {
+ boolean threw = true;
+ GarbageCollectorFileState fs = new GarbageCollectorFileState(root);
+ long candidates = 0;
+
+ try {
+ Stopwatch sw = Stopwatch.createStarted();
+ LOG.info("Starting blob consistency check");
+
+ // Find all blobs available in the blob store
+ ListenableFutureTask<Integer> blobIdRetriever = ListenableFutureTask.create(new BlobIdRetriever(fs));
+ executor.execute(blobIdRetriever);
+
+ // Mark all used blob references
+ iterateNodeTree(fs);
+
+ try {
+ blobIdRetriever.get();
+ } catch (ExecutionException e) {
+ LOG.warn("Error occurred while fetching all the blobIds from the BlobStore");
+ threw = false;
+ throw e;
+ }
+
+ LOG.trace("Starting difference phase of the consistency check");
+ FileLineDifferenceIterator iter = new FileLineDifferenceIterator(fs.getAvailableRefs(), fs.getMarkedRefs());
+ candidates = calculateDifference(fs, iter);
+ LOG.trace("Ending difference phase of the consistency check");
+
+ if (candidates > 0) {
+ LOG.warn("Consistency check failure in the the blob store : {}, check missing candidates in file {}",
+ blobStore, fs.getGcCandidates().getAbsolutePath());
+ }
+ } finally {
+ if (!LOG.isTraceEnabled() || candidates == 0) {
+ Closeables.close(fs, threw);
+ }
+ }
+ return candidates;
+ }
/**
* BlobIdRetriever class to retrieve all blob ids.
*/
@@ -578,7 +641,11 @@ public class MarkSweepGarbageCollector i
LineIterator.closeQuietly(marked);
LineIterator.closeQuietly(all);
}
-
+
+ private String getKey(String row) {
+ return row.split(DELIM)[0];
+ }
+
private String computeNextDiff() {
if (!all.hasNext()) {
return null;
@@ -594,7 +661,7 @@ public class MarkSweepGarbageCollector i
diff = all.next();
while (peekMarked.hasNext()) {
String marked = peekMarked.peek();
- int comparisonResult = diff.compareTo(marked);
+ int comparisonResult = getKey(diff).compareTo(getKey(marked));
if (comparisonResult > 0) {
//Extra entries in marked. Ignore them and move on
peekMarked.next();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferenceCollector.java Fri Sep 11 04:13:15 2015
@@ -18,14 +18,21 @@
*/
package org.apache.jackrabbit.oak.plugins.blob;
+import javax.annotation.Nullable;
+
/**
* Callback interface for collecting all blob references that are
* potentially accessible. Useful for marking referenced blobs as
* in use when collecting garbage in an external data store.
*/
public interface ReferenceCollector {
-
- void addReference(String reference);
-
+
+ /**
+ * Adds the reference detected with the node Id.
+ *
+ * @param reference
+ * @param nodeId
+ */
+ void addReference(String reference, @Nullable String nodeId);
}
Added: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java?rev=1702371&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java Fri Sep 11 04:13:15 2015
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.blob;
+
+import org.apache.jackrabbit.oak.api.Blob;
+
+/**
+ * Exposes the blob along with the Node id from which referenced
+ */
+public class ReferencedBlob {
+ private Blob blob;
+
+ private String id;
+
+ public ReferencedBlob(Blob blob, String id) {
+ this.setBlob(blob);
+ this.setId(id);
+ }
+
+ public Blob getBlob() {
+ return blob;
+ }
+
+ public void setBlob(Blob blob) {
+ this.blob = blob;
+ }
+
+ public String getId() {
+ return id;
+ }
+
+ public void setId(String id) {
+ this.id = id;
+ }
+
+ @Override
+ public String toString() {
+ return "ReferencedBlob{" +
+ "blob=" + blob +
+ ", id='" + id + '\'' +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ ReferencedBlob that = (ReferencedBlob) o;
+
+ if (!getBlob().equals(that.getBlob())) {
+ return false;
+ }
+ return !(getId() != null ? !getId().equals(that.getId()) : that.getId() != null);
+
+ }
+
+ @Override
+ public int hashCode() {
+ int result = getBlob().hashCode();
+ result = 31 * result + (getId() != null ? getId().hashCode() : 0);
+ return result;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/ReferencedBlob.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobCollector.java Fri Sep 11 04:13:15 2015
@@ -27,6 +27,7 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.commons.json.JsopReader;
import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
public class BlobCollector {
@@ -36,7 +37,7 @@ public class BlobCollector {
this.nodeStore = nodeStore;
}
- public void collect(NodeDocument doc, Collection<Blob> blobs) {
+ public void collect(NodeDocument doc, Collection<ReferencedBlob> blobs) {
for (String key : doc.keySet()) {
if (!Utils.isPropertyName(key)) {
continue;
@@ -44,13 +45,13 @@ public class BlobCollector {
Map<Revision, String> valueMap = doc.getLocalMap(key);
for (String v : valueMap.values()) {
if (v != null) {
- loadValue(v, blobs);
+ loadValue(v, blobs, doc.getPath());
}
}
}
}
- private void loadValue(String v, Collection<Blob> blobs) {
+ private void loadValue(String v, Collection<ReferencedBlob> blobs, String nodeId) {
JsopReader reader = new JsopTokenizer(v);
PropertyState p;
if (reader.matches('[')) {
@@ -58,14 +59,14 @@ public class BlobCollector {
if (p.getType() == Type.BINARIES) {
for (int i = 0; i < p.count(); i++) {
Blob b = p.getValue(Type.BINARY, i);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, nodeId));
}
}
} else {
p = DocumentPropertyState.readProperty("x", nodeStore, reader);
if (p.getType() == Type.BINARY) {
Blob b = p.getValue(Type.BINARY);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, nodeId));
}
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIterator.java Fri Sep 11 04:13:15 2015
@@ -21,7 +21,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.NoSuchElementException;
-import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
/**
* An iterator over all referenced binaries.
@@ -30,13 +30,13 @@ import org.apache.jackrabbit.oak.api.Blo
* The items are returned in no particular order.
* An item might be returned multiple times.
*/
-public class BlobReferenceIterator implements Iterator<Blob> {
+public class BlobReferenceIterator implements Iterator<ReferencedBlob> {
private static final int BATCH_SIZE = 1000;
private final DocumentStore docStore;
private final BlobCollector blobCollector;
- private HashSet<Blob> batch = new HashSet<Blob>();
- private Iterator<Blob> batchIterator;
+ private HashSet<ReferencedBlob> batch = new HashSet<ReferencedBlob>();
+ private Iterator<ReferencedBlob> batchIterator;
private boolean done;
private String fromKey = NodeDocument.MIN_ID_VALUE;
@@ -55,7 +55,7 @@ public class BlobReferenceIterator imple
}
@Override
- public Blob next() {
+ public ReferencedBlob next() {
// this will load the next batch if required
if (!hasNext()) {
throw new NoSuchElementException();
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentBlobReferenceRetriever.java Fri Sep 11 04:13:15 2015
@@ -26,6 +26,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,23 +44,24 @@ public class DocumentBlobReferenceRetrie
@Override
public void collectReferences(ReferenceCollector collector) {
int referencesFound = 0;
- Iterator<Blob> blobIterator = nodeStore.getReferencedBlobsIterator();
+ Iterator<ReferencedBlob> blobIterator = nodeStore.getReferencedBlobsIterator();
try {
while (blobIterator.hasNext()) {
- Blob blob = blobIterator.next();
+ ReferencedBlob refBlob = blobIterator.next();
+ Blob blob = refBlob.getBlob();
referencesFound++;
//TODO this mode would also add in memory blobId
//Would that be an issue
if (blob instanceof BlobStoreBlob) {
- collector.addReference(((BlobStoreBlob) blob).getBlobId());
+ collector.addReference(((BlobStoreBlob) blob).getBlobId(), refBlob.getId());
} else {
//TODO Should not rely on toString. Instead obtain
//secure reference and convert that to blobId using
//blobStore
- collector.addReference(blob.toString());
+ collector.addReference(blob.toString(), refBlob.getId());
}
}
}finally{
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java Fri Sep 11 04:13:15 2015
@@ -81,6 +81,7 @@ import org.apache.jackrabbit.oak.commons
import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.MarkSweepGarbageCollector;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.Checkpoints.Info;
import org.apache.jackrabbit.oak.plugins.document.cache.CacheInvalidationStats;
import org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobReferenceIterator;
@@ -2686,7 +2687,7 @@ public final class DocumentNodeStore
* @see org.apache.jackrabbit.oak.plugins.document.mongo.MongoBlobReferenceIterator
* @return an iterator for all the blobs
*/
- public Iterator<Blob> getReferencedBlobsIterator() {
+ public Iterator<ReferencedBlob> getReferencedBlobsIterator() {
if(store instanceof MongoDocumentStore){
return new MongoBlobReferenceIterator(this, (MongoDocumentStore) store);
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/document/mongo/MongoBlobReferenceIterator.java Fri Sep 11 04:13:15 2015
@@ -28,16 +28,16 @@ import com.mongodb.DBCollection;
import com.mongodb.DBCursor;
import com.mongodb.DBObject;
import com.mongodb.QueryBuilder;
-import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.BlobCollector;
import org.apache.jackrabbit.oak.plugins.document.Collection;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
-public class MongoBlobReferenceIterator extends AbstractIterator<Blob> implements Closeable {
+public class MongoBlobReferenceIterator extends AbstractIterator<ReferencedBlob> implements Closeable {
private final MongoDocumentStore documentStore;
private final BlobCollector blobCollector;
- private final Queue<Blob> blobs = Queues.newArrayDeque();
+ private final Queue<ReferencedBlob> blobs = Queues.newArrayDeque();
private DBCursor cursor;
@@ -48,7 +48,7 @@ public class MongoBlobReferenceIterator
}
@Override
- protected Blob computeNext() {
+ protected ReferencedBlob computeNext() {
if (blobs.isEmpty()) {
loadBatch();
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/Segment.java Fri Sep 11 04:13:15 2015
@@ -348,7 +348,7 @@ public class Segment {
for (int i = 0; i < blobrefcount; i++) {
int offset = (data.getShort(blobrefpos + i * 2) & 0xffff) << 2;
SegmentBlob blob = new SegmentBlob(new RecordId(id, offset));
- collector.addReference(blob.getBlobId());
+ collector.addReference(blob.getBlobId(), null);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobCollectorTest.java Fri Sep 11 04:13:15 2015
@@ -25,6 +25,7 @@ import java.util.List;
import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.plugins.document.util.Utils;
import org.apache.jackrabbit.oak.plugins.memory.PropertyBuilder;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
@@ -48,7 +49,7 @@ public class BlobCollectorTest {
@Test
public void testCollect() throws Exception {
NodeBuilder b1 = store.getRoot().builder();
- List<Blob> blobs = Lists.newArrayList();
+ List<ReferencedBlob> blobs = Lists.newArrayList();
b1.child("x").child("y");
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
@@ -58,7 +59,7 @@ public class BlobCollectorTest {
b1 = store.getRoot().builder();
Blob b = store.createBlob(randomStream(i, 4096));
b1.child("x").child("y").setProperty("b" + i, b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y"));
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
@@ -68,7 +69,7 @@ public class BlobCollectorTest {
for(int i = 0; i < 2; i++){
Blob b = store.createBlob(randomStream(i, 4096));
p1.addValue(b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y"));
}
b1 = store.getRoot().builder();
b1.child("x").child("y").setProperty(p1.getPropertyState());
@@ -80,16 +81,16 @@ public class BlobCollectorTest {
//Change the see to create diff binary
Blob b = store.createBlob(randomStream(i+1, 4096));
b1.child("x").child("y").setProperty("b" + i, b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y"));
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
NodeDocument doc =
store.getDocumentStore().find(Collection.NODES, Utils.getIdFromPath("/x/y"));
- List<Blob> collectedBlobs = Lists.newArrayList();
+ List<ReferencedBlob> collectedBlobs = Lists.newArrayList();
blobCollector.collect(doc, collectedBlobs);
assertEquals(blobs.size(), collectedBlobs.size());
- assertEquals(new HashSet<Blob>(blobs), new HashSet<Blob>(collectedBlobs));
+ assertEquals(new HashSet<ReferencedBlob>(blobs), new HashSet<ReferencedBlob>(collectedBlobs));
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceIteratorTest.java Fri Sep 11 04:13:15 2015
@@ -26,6 +26,7 @@ import java.util.List;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -76,19 +77,19 @@ public class BlobReferenceIteratorTest {
@Test
public void testBlobIterator() throws Exception{
- List<Blob> blobs = Lists.newArrayList();
+ List<ReferencedBlob> blobs = Lists.newArrayList();
//1. Set some single value Binary property
for(int i = 0; i < 10; i++){
NodeBuilder b1 = store.getRoot().builder();
Blob b = store.createBlob(randomStream(i, 4096));
b1.child("x").child("y"+1).setProperty("b" + i, b);
- blobs.add(b);
+ blobs.add(new ReferencedBlob(b, "/x/y" + 1));
store.merge(b1, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
- List<Blob> collectedBlobs = ImmutableList.copyOf(store.getReferencedBlobsIterator());
+ List<ReferencedBlob> collectedBlobs = ImmutableList.copyOf(store.getReferencedBlobsIterator());
assertEquals(blobs.size(), collectedBlobs.size());
- assertEquals(new HashSet<Blob>(blobs), new HashSet<Blob>(collectedBlobs));
+ assertEquals(new HashSet<ReferencedBlob>(blobs), new HashSet<ReferencedBlob>(collectedBlobs));
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/BlobReferenceTest.java Fri Sep 11 04:13:15 2015
@@ -25,6 +25,7 @@ import java.util.Iterator;
import java.util.Random;
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.plugins.blob.ReferencedBlob;
import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
@@ -42,13 +43,13 @@ public class BlobReferenceTest {
HashSet<String> set = new HashSet<String>();
for (int i = 0; i < 100; i++) {
Blob b = a.createBlob(randomStream(i, 10));
- set.add(b.toString());
+ set.add(new ReferencedBlob(b, "/c" + i).toString());
a.child("c" + i).setProperty("x", b);
}
s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
- Iterator<Blob> it = s.getReferencedBlobsIterator();
+ Iterator<ReferencedBlob> it = s.getReferencedBlobsIterator();
while (it.hasNext()) {
- Blob b = it.next();
+ ReferencedBlob b = it.next();
set.remove(b.toString());
}
assertTrue(set.isEmpty());
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java Fri Sep 11 04:13:15 2015
@@ -31,6 +31,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.mongodb.BasicDBObject;
@@ -169,8 +170,59 @@ public class MongoBlobGCTest extends Abs
Set<String> existingAfterGC = gc(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
}
+
+ @Test
+ public void consistencyCheckInit() throws Exception {
+ DataStoreState state = setUp(true);
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithGc() throws Exception {
+ DataStoreState state = setUp(true);
+ Set<String> existingAfterGC = gc(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithRenegadeDelete() throws Exception {
+ DataStoreState state = setUp(true);
+
+ // Simulate faulty state by deleting some blobs directly
+ Random rand = new Random(87);
+ List<String> existing = Lists.newArrayList(state.blobsPresent);
+
+ GarbageCollectableBlobStore store = (GarbageCollectableBlobStore)
+ mk.getNodeStore().getBlobStore();
+ long count = store.countDeleteChunks(ImmutableList.of(existing.get(rand.nextInt(existing.size()))), 0);
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(count, candidates);
+ }
private Set<String> gc(int blobGcMaxAgeInSecs) throws Exception {
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gc = init(blobGcMaxAgeInSecs, executor);
+ gc.collectGarbage(false);
+
+ assertEquals(0, executor.getTaskCount());
+ return iterate();
+ }
+
+ private MarkSweepGarbageCollector init(int blobGcMaxAgeInSecs, ThreadPoolExecutor executor) throws Exception {
DocumentNodeStore store = mk.getNodeStore();
String repoId = null;
if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
@@ -179,14 +231,10 @@ public class MongoBlobGCTest extends Abs
new ByteArrayInputStream(new byte[0]),
REPOSITORY.getNameFromId(repoId));
}
- ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
new DocumentBlobReferenceRetriever(store),
(GarbageCollectableBlobStore) store.getBlobStore(), executor, "./target", 5, blobGcMaxAgeInSecs, repoId);
- gc.collectGarbage(false);
-
- assertEquals(0, executor.getTaskCount());
- return iterate();
+ return gc;
}
protected Set<String> iterate() throws Exception {
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobIT.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ExternalBlobIT.java Fri Sep 11 04:13:15 2015
@@ -118,7 +118,7 @@ public class ExternalBlobIT {
final List<String> refrences = Lists.newArrayList();
store.getTracker().collectBlobReferences(new ReferenceCollector() {
@Override
- public void addReference(String reference) {
+ public void addReference(String reference, String nodeId) {
assertNotNull(reference);
refrences.add(reference);
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1702371&r1=1702370&r2=1702371&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java Fri Sep 11 04:13:15 2015
@@ -226,20 +226,49 @@ public class SegmentDataStoreBlobGCIT {
assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty());
}
- private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
- String repoId = null;
- if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
- repoId = ClusterRepositoryInfo.createId(nodeStore);
- ((SharedDataStore) store.getBlobStore()).addMetadataRecord(
- new ByteArrayInputStream(new byte[0]),
- REPOSITORY.getNameFromId(repoId));
- }
+ @Test
+ public void consistencyCheckInit() throws Exception {
+ DataStoreState state = setUp();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithGc() throws Exception {
+ DataStoreState state = setUp();
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
- MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
- new SegmentBlobReferenceRetriever(store.getTracker()),
- (GarbageCollectableBlobStore) store.getBlobStore(), executor, "./target", 2048, maxBlobGcInSecs,
- repoId);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
+ @Test
+ public void consistencyCheckWithRenegadeDelete() throws Exception {
+ DataStoreState state = setUp();
+
+ // Simulate faulty state by deleting some blobs directly
+ Random rand = new Random(87);
+ List<String> existing = Lists.newArrayList(state.blobsPresent);
+
+ long count = blobStore.countDeleteChunks(ImmutableList.of(existing.get(rand.nextInt(existing.size()))), 0);
+
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(count, candidates);
+ }
+
+ private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gc = init(maxBlobGcInSecs, executor);
gc.collectGarbage(false);
assertEquals(0, executor.getTaskCount());
@@ -247,6 +276,21 @@ public class SegmentDataStoreBlobGCIT {
log.info("{} blobs existing after gc : {}", existingAfterGC.size(), existingAfterGC);
return existingAfterGC;
}
+
+ private MarkSweepGarbageCollector init(long blobGcMaxAgeInSecs, ThreadPoolExecutor executor) throws Exception {
+ String repoId = null;
+ if (SharedDataStoreUtils.isShared(store.getBlobStore())) {
+ repoId = ClusterRepositoryInfo.createId(nodeStore);
+ ((SharedDataStore) store.getBlobStore()).addMetadataRecord(
+ new ByteArrayInputStream(new byte[0]),
+ REPOSITORY.getNameFromId(repoId));
+ }
+ MarkSweepGarbageCollector gc = new MarkSweepGarbageCollector(
+ new SegmentBlobReferenceRetriever(store.getTracker()),
+ (GarbageCollectableBlobStore) store.getBlobStore(), executor, "./target", 2048, blobGcMaxAgeInSecs,
+ repoId);
+ return gc;
+ }
protected Set<String> iterate() throws Exception {
Iterator<String> cur = blobStore.getAllChunkIds(0);