You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/06/09 05:45:58 UTC
svn commit: r1747494 - in /jackrabbit/oak/branches/1.2/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/
test/java/org/apache/jackrabbit/oak/plugins/document/
test/java/org/apache/jackrabbit/oak/plugins/segment/
Author: amitj
Date: Thu Jun 9 05:45:57 2016
New Revision: 1747494
URL: http://svn.apache.org/viewvc?rev=1747494&view=rev
Log:
OAK-4441: [BlobGC] Writing of strings should be escaped
Merged r1747492 from trunk
Modified:
jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
Modified: jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1747494&r1=1747493&r2=1747494&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Thu Jun 9 05:45:57 2016
@@ -57,6 +57,7 @@ import org.apache.commons.io.LineIterato
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -449,8 +450,10 @@ public class MarkSweepGarbageCollector i
* Save batch to file.
*/
static void saveBatchToFile(List<String> ids, BufferedWriter writer) throws IOException {
- writer.append(Joiner.on(NEWLINE).join(ids));
- writer.append(NEWLINE);
+ for (String id : ids) {
+ ExternalSort.writeLine(writer, id);
+ writer.append(NEWLINE);
+ }
ids.clear();
writer.flush();
}
Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1747494&r1=1747493&r2=1747494&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java Thu Jun 9 05:45:57 2016
@@ -144,6 +144,24 @@ public class MongoBlobGCTest extends Abs
return set;
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ DocumentNodeStore s = mk.getNodeStore();
+ NodeBuilder a = s.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ Blob b = s.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q\\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter =
+ ((GarbageCollectableBlobStore) s.getBlobStore())
+ .resolveChunks(b.toString());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
private void deleteFromMongo(String nodeId) {
DBCollection coll = mongoConnection.getDB().getCollection("nodes");
BasicDBObject blobNodeObj = new BasicDBObject();
@@ -157,6 +175,17 @@ public class MongoBlobGCTest extends Abs
Set<String> existingAfterGC = gc(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
}
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp(true);
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gc(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void noGc() throws Exception {
Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1747494&r1=1747493&r2=1747494&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Thu Jun 9 05:45:57 2016
@@ -135,6 +135,22 @@ public class SharedBlobStoreGCTest {
}
@Test
+ public void testGCWithNodeSpecialChars() throws Exception {
+ log.debug("Running testGC()");
+ // Only run the mark phase on both the clusters
+ cluster1.initBlobs.addAll(cluster1.addNodeSpecialChars());
+ cluster2.initBlobs.addAll(cluster1.addNodeSpecialChars());
+ cluster1.gc.collectGarbage(true);
+ cluster2.gc.collectGarbage(true);
+
+ // Execute the gc with sweep
+ cluster1.gc.collectGarbage(false);
+
+ Assert.assertEquals(true, Sets.symmetricDifference(Sets.union(cluster1.getInitBlobs(), cluster2.getInitBlobs()),
+ cluster1.getExistingBlobIds()).isEmpty());
+ }
+
+ @Test
public void testGCStats() throws Exception {
log.debug("Running testGCStats()");
// Only run the mark phase on both the clusters to get the stats
@@ -284,6 +300,24 @@ public class SharedBlobStoreGCTest {
}
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = ds.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ Blob b = ds.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q\\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter =
+ ((GarbageCollectableBlobStore) ds.getBlobStore())
+ .resolveChunks(b.toString());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ ds.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
+
public Set<String> getExistingBlobIds() throws Exception {
GarbageCollectableBlobStore store = (GarbageCollectableBlobStore) ds.getBlobStore();
Iterator<String> cur = store.getAllChunkIds(0);
Modified: jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java?rev=1747494&r1=1747493&r2=1747494&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java (original)
+++ jackrabbit/oak/branches/1.2/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java Thu Jun 9 05:45:57 2016
@@ -187,6 +187,22 @@ public class SegmentDataStoreBlobGCTest
nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
return set;
}
+
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = nodeStore.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q \\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
private class DataStoreState {
Set<String> blobsAdded = Sets.newHashSet();
Set<String> blobsPresent = Sets.newHashSet();
@@ -216,7 +232,17 @@ public class SegmentDataStoreBlobGCTest
Set<String> existingAfterGC = gcInternal(86400);
assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty());
}
-
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp();
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void consistencyCheckInit() throws Exception {
DataStoreState state = setUp();