You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/01/25 14:20:03 UTC
svn commit: r1726621 - in /jackrabbit/oak/trunk:
oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-core...
Author: amitj
Date: Mon Jan 25 13:20:03 2016
New Revision: 1726621
URL: http://svn.apache.org/viewvc?rev=1726621&view=rev
Log:
OAK-3921: DataStoreBlobStore - Limit resolveChunks only to non inlined blobs
* DataStoreBlobStore#resolveChunks only resolves to blob ids stored in the DataStore.
* Removed usage of InMemoryDataRecord outside the package
Modified:
jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Modified: jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java?rev=1726621&r1=1726620&r2=1726621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-blob/src/main/java/org/apache/jackrabbit/oak/spi/blob/GarbageCollectableBlobStore.java Mon Jan 25 13:20:03 2016
@@ -111,7 +111,8 @@ public interface GarbageCollectableBlobS
long countDeleteChunks(List<String> chunkIds, long maxLastModifiedTime) throws Exception;
/**
- * Resolve chunks from the given Id.
+ * Resolve chunks stored in the blob store from the given Id.
+ * This will not return any chunks stored in-line in the id.
*
* @param blobId the blob id
* @return the iterator
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1726621&r1=1726620&r2=1726621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Mon Jan 25 13:20:03 2016
@@ -57,7 +57,6 @@ import org.apache.commons.io.LineIterato
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.IOUtils;
-import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -704,12 +703,7 @@ public class MarkSweepGarbageCollector i
} else {
//This entry is not found in marked entries
//hence part of diff
- if (!InMemoryDataRecord.isInstance(getKey(diff))) {
- return diff;
- } else {
- diff = null;
- break;
- }
+ return diff;
}
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java?rev=1726621&r1=1726620&r2=1726621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreBlobStore.java Mon Jan 25 13:20:03 2016
@@ -424,7 +424,10 @@ public class DataStoreBlobStore implemen
@Override
public Iterator<String> resolveChunks(String blobId) throws IOException {
- return Iterators.singletonIterator(blobId);
+ if (!InMemoryDataRecord.isInstance(blobId)) {
+ return Iterators.singletonIterator(blobId);
+ }
+ return Iterators.emptyIterator();
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java?rev=1726621&r1=1726620&r2=1726621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/InMemoryDataRecord.java Mon Jan 25 13:20:03 2016
@@ -31,7 +31,7 @@ import org.slf4j.LoggerFactory;
/**
* Represents binary data which is backed by a byte[] (in memory).
*/
-public class InMemoryDataRecord implements DataRecord {
+class InMemoryDataRecord implements DataRecord {
/**
* Logger instance for this class
@@ -89,7 +89,7 @@ public class InMemoryDataRecord implemen
* @param id DataRecord identifier
* @return true if it can be converted
*/
- public static boolean isInstance(String id) {
+ static boolean isInstance(String id) {
return id.startsWith(PREFIX);
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java?rev=1726621&r1=1726620&r2=1726621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileLineDifferenceIteratorTest.java Mon Jan 25 13:20:03 2016
@@ -95,8 +95,8 @@ public class FileLineDifferenceIteratorT
assertReverseDiff("a,b,c,e,h", "a,b,c", asList("e", "h"));
assertReverseDiff("a,b,d,e", "a,b,c", asList("d", "e"));
assertReverseDiff("a,b,d", "a,b,d", Collections.<String>emptyList());
- assertReverseDiff("a,0xb,d,e,f", "a,d", asList("e", "f"));
- assertReverseDiff("a,0xb,d,e,f", "a,d,e,f,g", Collections.<String>emptyList());
+ assertReverseDiff("a,0xb,d,e,f", "a,d", asList("0xb", "e", "f"));
+ assertReverseDiff("a,0xb,d,e,f", "a,d,e,f,g", asList("0xb"));
}
private static void assertReverseDiff(String marked, String all, List<String> diff) throws IOException {
Modified: jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1726621&r1=1726620&r2=1726621&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java Mon Jan 25 13:20:03 2016
@@ -52,6 +52,7 @@ import com.google.common.collect.Sets;
import com.google.common.io.Closeables;
import org.apache.commons.io.FileUtils;
+import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.GarbageCollectorFileState;
@@ -130,7 +131,9 @@ public class SegmentDataStoreBlobGCIT {
}
public DataStoreState setUp() throws Exception {
- blobStore = DataStoreUtils.getBlobStore();
+ if (blobStore == null) {
+ blobStore = DataStoreUtils.getBlobStore();
+ }
nodeStore = getNodeStore(blobStore);
startDate = new Date();
@@ -174,7 +177,7 @@ public class SegmentDataStoreBlobGCIT {
DataStoreState state = new DataStoreState();
for (int i = 0; i < numBlobs; i++) {
- SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 16516));
+ SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 18342));
Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
while (idIter.hasNext()) {
String chunk = idIter.next();
@@ -201,7 +204,19 @@ public class SegmentDataStoreBlobGCIT {
return state;
}
-
+
+ private HashSet<String> addInlined() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = nodeStore.getRoot().builder();
+ int number = 4;
+ for (int i = 0; i < number; i++) {
+ Blob b = nodeStore.createBlob(randomStream(i, 16514));
+ a.child("cinline" + i).setProperty("x", b);
+ }
+ nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
private class DataStoreState {
Set<String> blobsAdded = Sets.newHashSet();
Set<String> blobsPresent = Sets.newHashSet();
@@ -296,7 +311,30 @@ public class SegmentDataStoreBlobGCIT {
assertTrue(Sets.difference(state.blobsPresent, existingAfterGC).isEmpty());
assertEquals(gc.additionalBlobs, Sets.symmetricDifference(state.blobsPresent, existingAfterGC));
}
-
+
+ @Test
+ public void gcWithInlined() throws Exception {
+ blobStore = new DataStoreBlobStore(DataStoreUtils.createFDS(new File(getWorkDir(), "datastore"), 16516));
+ DataStoreState state = setUp();
+ addInlined();
+ log.info("{} blobs that should remain after gc : {}", state.blobsAdded.size(), state.blobsAdded);
+ log.info("{} blobs for nodes which are deleted : {}", state.blobsPresent.size(), state.blobsPresent);
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
+ @Test
+ public void consistencyCheckInlined() throws Exception {
+ blobStore = new DataStoreBlobStore(DataStoreUtils.createFDS(new File(getWorkDir(), "datastore"), 16516));
+ DataStoreState state = setUp();
+ addInlined();
+ ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
+ MarkSweepGarbageCollector gcObj = init(86400, executor);
+ long candidates = gcObj.checkConsistency();
+ assertEquals(1, executor.getTaskCount());
+ assertEquals(0, candidates);
+ }
+
private Set<String> gcInternal(long maxBlobGcInSecs) throws Exception {
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gc = init(maxBlobGcInSecs, executor);