You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/06/09 04:43:51 UTC
svn commit: r1747492 - in /jackrabbit/oak/trunk:
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/ oak-run...
Author: amitj
Date: Thu Jun 9 04:43:50 2016
New Revision: 1747492
URL: http://svn.apache.org/viewvc?rev=1747492&view=rev
Log:
OAK-4441: [BlobGC] Writing of strings should be escaped
Escaped strings when writing to the file.
Modified:
jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Modified: jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java (original)
+++ jackrabbit/oak/trunk/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java Thu Jun 9 04:43:50 2016
@@ -629,11 +629,11 @@ public class ExternalSort {
}
};
- static String readLine(BufferedReader br) throws IOException {
+ public static String readLine(BufferedReader br) throws IOException {
return EscapeUtils.unescapeLineBreaks(br.readLine());
}
- static void writeLine(BufferedWriter wr, String line) throws IOException {
+ public static void writeLine(BufferedWriter wr, String line) throws IOException {
wr.write(EscapeUtils.escapeLineBreak(line));
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Thu Jun 9 04:43:50 2016
@@ -57,6 +57,7 @@ import org.apache.commons.io.LineIterato
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -453,8 +454,10 @@ public class MarkSweepGarbageCollector i
* Save batch to file.
*/
static void saveBatchToFile(List<String> ids, BufferedWriter writer) throws IOException {
- writer.append(Joiner.on(NEWLINE).join(ids));
- writer.append(NEWLINE);
+ for (String id : ids) {
+ ExternalSort.writeLine(writer, id);
+ writer.append(NEWLINE);
+ }
ids.clear();
writer.flush();
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java Thu Jun 9 04:43:50 2016
@@ -147,6 +147,24 @@ public class MongoBlobGCTest extends Abs
return set;
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ DocumentNodeStore s = mk.getNodeStore();
+ NodeBuilder a = s.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ Blob b = s.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q\\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter =
+ ((GarbageCollectableBlobStore) s.getBlobStore())
+ .resolveChunks(b.toString());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
private void deleteFromMongo(String nodeId) {
DBCollection coll = mongoConnection.getDB().getCollection("nodes");
BasicDBObject blobNodeObj = new BasicDBObject();
@@ -160,6 +178,17 @@ public class MongoBlobGCTest extends Abs
Set<String> existingAfterGC = gc(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
}
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp(true);
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gc(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void noGc() throws Exception {
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Thu Jun 9 04:43:50 2016
@@ -135,6 +135,22 @@ public class SharedBlobStoreGCTest {
}
@Test
+ public void testGCWithNodeSpecialChars() throws Exception {
+ log.debug("Running testGC()");
+ // Only run the mark phase on both the clusters
+ cluster1.initBlobs.addAll(cluster1.addNodeSpecialChars());
+ cluster2.initBlobs.addAll(cluster1.addNodeSpecialChars());
+ cluster1.gc.collectGarbage(true);
+ cluster2.gc.collectGarbage(true);
+
+ // Execute the gc with sweep
+ cluster1.gc.collectGarbage(false);
+
+ Assert.assertEquals(true, Sets.symmetricDifference(Sets.union(cluster1.getInitBlobs(), cluster2.getInitBlobs()),
+ cluster1.getExistingBlobIds()).isEmpty());
+ }
+
+ @Test
public void testGCStats() throws Exception {
log.debug("Running testGCStats()");
// Only run the mark phase on both the clusters to get the stats
@@ -284,6 +300,24 @@ public class SharedBlobStoreGCTest {
}
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = ds.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ Blob b = ds.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q\\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter =
+ ((GarbageCollectableBlobStore) ds.getBlobStore())
+ .resolveChunks(b.toString());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ ds.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
+
public Set<String> getExistingBlobIds() throws Exception {
GarbageCollectableBlobStore store = (GarbageCollectableBlobStore) ds.getBlobStore();
Iterator<String> cur = store.getAllChunkIds(0);
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java Thu Jun 9 04:43:50 2016
@@ -39,6 +39,7 @@ import joptsimple.OptionParser;
import joptsimple.OptionSet;
import joptsimple.OptionSpec;
import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
@@ -117,9 +118,11 @@ class DumpDataStoreReferencesCommand imp
idBatch.add(delimJoiner.join(id, nodeId));
count.getAndIncrement();
if (idBatch.size() >= 1024) {
- writer.append(Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).join(idBatch));
- writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
- writer.flush();
+ for (String rec : idBatch) {
+ ExternalSort.writeLine(writer, rec);
+ writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
+ writer.flush();
+ }
idBatch.clear();
}
}
@@ -130,9 +133,11 @@ class DumpDataStoreReferencesCommand imp
}
);
if (!idBatch.isEmpty()) {
- writer.append(Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).join(idBatch));
- writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
- writer.flush();
+ for (String rec : idBatch) {
+ ExternalSort.writeLine(writer, rec);
+ writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
+ writer.flush();
+ }
idBatch.clear();
}
System.out.println(count.get() + " DataStore references dumped in " + dumpFile);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/SegmentDataStoreBlobGCIT.java Thu Jun 9 04:43:50 2016
@@ -207,6 +207,21 @@ public class SegmentDataStoreBlobGCIT {
return set;
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = nodeStore.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q \\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
private class DataStoreState {
Set<String> blobsAdded = Sets.newHashSet();
Set<String> blobsPresent = Sets.newHashSet();
@@ -236,7 +251,17 @@ public class SegmentDataStoreBlobGCIT {
Set<String> existingAfterGC = gcInternal(86400);
assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty());
}
-
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp();
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void consistencyCheckInit() throws Exception {
DataStoreState state = setUp();
Modified: jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1747492&r1=1747491&r2=1747492&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java (original)
+++ jackrabbit/oak/trunk/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java Thu Jun 9 04:43:50 2016
@@ -219,6 +219,20 @@ public class SegmentDataStoreBlobGCIT {
nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
return set;
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = nodeStore.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q \\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
private class DataStoreState {
Set<String> blobsAdded = Sets.newHashSet();
@@ -249,7 +263,17 @@ public class SegmentDataStoreBlobGCIT {
Set<String> existingAfterGC = gcInternal(86400);
assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty());
}
-
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp();
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void consistencyCheckInit() throws Exception {
DataStoreState state = setUp();