You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/06/09 05:18:02 UTC
svn commit: r1747493 - in /jackrabbit/oak/branches/1.4: ./
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/documen...
Author: amitj
Date: Thu Jun 9 05:18:02 2016
New Revision: 1747493
URL: http://svn.apache.org/viewvc?rev=1747493&view=rev
Log:
OAK-4441: [BlobGC] Writing of strings should be escaped
Merged r1747492 from trunk
Modified:
jackrabbit/oak/branches/1.4/ (props changed)
jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java
jackrabbit/oak/branches/1.4/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
Propchange: jackrabbit/oak/branches/1.4/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 05:18:02 2016
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746696,1746981
+/jackrabbit/oak/trunk:1733615,1733875,1733913,1733929,1734230,1734254,1734279,1734941,1735052,1735405,1735484,1735549,1735564,1735588,1735622,1735638,1735919,1735983,1736176,1737309-1737310,1737334,1737349,1737998,1738004,1738775,1738795,1738833,1738950,1738957,1738963,1739894,1740116,1740625-1740626,1740971,1741032,1741339,1741343,1742520,1742888,1742916,1743097,1743172,1743343,1744265,1744959,1745038,1745197,1745368,1746086,1746117,1746342,1746696,1746981,1747492
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java?rev=1747493&r1=1747492&r2=1747493&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java (original)
+++ jackrabbit/oak/branches/1.4/oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/sort/ExternalSort.java Thu Jun 9 05:18:02 2016
@@ -629,11 +629,11 @@ public class ExternalSort {
}
};
- static String readLine(BufferedReader br) throws IOException {
+ public static String readLine(BufferedReader br) throws IOException {
return EscapeUtils.unescapeLineBreaks(br.readLine());
}
- static void writeLine(BufferedWriter wr, String line) throws IOException {
+ public static void writeLine(BufferedWriter wr, String line) throws IOException {
wr.write(EscapeUtils.escapeLineBreak(line));
}
Modified: jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java?rev=1747493&r1=1747492&r2=1747493&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/MarkSweepGarbageCollector.java Thu Jun 9 05:18:02 2016
@@ -57,6 +57,7 @@ import org.apache.commons.io.LineIterato
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStoreException;
import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils;
import org.apache.jackrabbit.oak.plugins.blob.datastore.SharedDataStoreUtils.SharedStoreRecordType;
import org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore;
@@ -449,8 +450,10 @@ public class MarkSweepGarbageCollector i
* Save batch to file.
*/
static void saveBatchToFile(List<String> ids, BufferedWriter writer) throws IOException {
- writer.append(Joiner.on(NEWLINE).join(ids));
- writer.append(NEWLINE);
+ for (String id : ids) {
+ ExternalSort.writeLine(writer, id);
+ writer.append(NEWLINE);
+ }
ids.clear();
writer.flush();
}
Modified: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java?rev=1747493&r1=1747492&r2=1747493&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/MongoBlobGCTest.java Thu Jun 9 05:18:02 2016
@@ -147,6 +147,24 @@ public class MongoBlobGCTest extends Abs
return set;
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ DocumentNodeStore s = mk.getNodeStore();
+ NodeBuilder a = s.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ Blob b = s.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q\\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter =
+ ((GarbageCollectableBlobStore) s.getBlobStore())
+ .resolveChunks(b.toString());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
private void deleteFromMongo(String nodeId) {
DBCollection coll = mongoConnection.getDB().getCollection("nodes");
BasicDBObject blobNodeObj = new BasicDBObject();
@@ -160,6 +178,17 @@ public class MongoBlobGCTest extends Abs
Set<String> existingAfterGC = gc(0);
assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
}
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp(true);
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gc(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void noGc() throws Exception {
Modified: jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java?rev=1747493&r1=1747492&r2=1747493&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java (original)
+++ jackrabbit/oak/branches/1.4/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/document/SharedBlobStoreGCTest.java Thu Jun 9 05:18:02 2016
@@ -135,6 +135,22 @@ public class SharedBlobStoreGCTest {
}
@Test
+ public void testGCWithNodeSpecialChars() throws Exception {
+ log.debug("Running testGC()");
+ // Only run the mark phase on both the clusters
+ cluster1.initBlobs.addAll(cluster1.addNodeSpecialChars());
+ cluster2.initBlobs.addAll(cluster1.addNodeSpecialChars());
+ cluster1.gc.collectGarbage(true);
+ cluster2.gc.collectGarbage(true);
+
+ // Execute the gc with sweep
+ cluster1.gc.collectGarbage(false);
+
+ Assert.assertEquals(true, Sets.symmetricDifference(Sets.union(cluster1.getInitBlobs(), cluster2.getInitBlobs()),
+ cluster1.getExistingBlobIds()).isEmpty());
+ }
+
+ @Test
public void testGCStats() throws Exception {
log.debug("Running testGCStats()");
// Only run the mark phase on both the clusters to get the stats
@@ -284,6 +300,24 @@ public class SharedBlobStoreGCTest {
}
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = ds.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ Blob b = ds.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q\\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter =
+ ((GarbageCollectableBlobStore) ds.getBlobStore())
+ .resolveChunks(b.toString());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ ds.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
+
+
public Set<String> getExistingBlobIds() throws Exception {
GarbageCollectableBlobStore store = (GarbageCollectableBlobStore) ds.getBlobStore();
Iterator<String> cur = store.getAllChunkIds(0);
Modified: jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java?rev=1747493&r1=1747492&r2=1747493&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java (original)
+++ jackrabbit/oak/branches/1.4/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DumpDataStoreReferencesCommand.java Thu Jun 9 05:18:02 2016
@@ -36,6 +36,7 @@ import com.mongodb.MongoClient;
import com.mongodb.MongoClientURI;
import com.mongodb.MongoURI;
import org.apache.jackrabbit.oak.commons.IOUtils;
+import org.apache.jackrabbit.oak.commons.sort.ExternalSort;
import org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever;
import org.apache.jackrabbit.oak.plugins.blob.ReferenceCollector;
import org.apache.jackrabbit.oak.plugins.document.DocumentBlobReferenceRetriever;
@@ -105,9 +106,11 @@ class DumpDataStoreReferencesCommand imp
idBatch.add(delimJoiner.join(id, nodeId));
count.getAndIncrement();
if (idBatch.size() >= 1024) {
- writer.append(Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).join(idBatch));
- writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
- writer.flush();
+ for (String rec : idBatch) {
+ ExternalSort.writeLine(writer, rec);
+ writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
+ writer.flush();
+ }
idBatch.clear();
}
}
@@ -118,9 +121,11 @@ class DumpDataStoreReferencesCommand imp
}
);
if (!idBatch.isEmpty()) {
- writer.append(Joiner.on(StandardSystemProperty.LINE_SEPARATOR.value()).join(idBatch));
- writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
- writer.flush();
+ for (String rec : idBatch) {
+ ExternalSort.writeLine(writer, rec);
+ writer.append(StandardSystemProperty.LINE_SEPARATOR.value());
+ writer.flush();
+ }
idBatch.clear();
}
System.out.println(count.get() + " DataStore references dumped in " + dumpFile);
Modified: jackrabbit/oak/branches/1.4/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.4/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java?rev=1747493&r1=1747492&r2=1747493&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.4/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java (original)
+++ jackrabbit/oak/branches/1.4/oak-segment/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCIT.java Thu Jun 9 05:18:02 2016
@@ -216,6 +216,20 @@ public class SegmentDataStoreBlobGCIT {
nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
return set;
}
+ private HashSet<String> addNodeSpecialChars() throws Exception {
+ HashSet<String> set = new HashSet<String>();
+ NodeBuilder a = nodeStore.getRoot().builder();
+ int number = 1;
+ for (int i = 0; i < number; i++) {
+ SegmentBlob b = (SegmentBlob) nodeStore.createBlob(randomStream(i, 18432));
+ NodeBuilder n = a.child("cspecial");
+ n.child("q \\%22afdg\\%22").setProperty("x", b);
+ Iterator<String> idIter = blobStore.resolveChunks(b.getBlobId());
+ set.addAll(Lists.newArrayList(idIter));
+ }
+ nodeStore.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ return set;
+ }
private class DataStoreState {
Set<String> blobsAdded = Sets.newHashSet();
@@ -246,7 +260,17 @@ public class SegmentDataStoreBlobGCIT {
Set<String> existingAfterGC = gcInternal(86400);
assertTrue(Sets.symmetricDifference(state.blobsAdded, existingAfterGC).isEmpty());
}
-
+
+ @Test
+ public void gcSpecialChar() throws Exception {
+ DataStoreState state = setUp();
+ Set<String> specialCharNodeBlobs = addNodeSpecialChars();
+ state.blobsAdded.addAll(specialCharNodeBlobs);
+ state.blobsPresent.addAll(specialCharNodeBlobs);
+ Set<String> existingAfterGC = gcInternal(0);
+ assertTrue(Sets.symmetricDifference(state.blobsPresent, existingAfterGC).isEmpty());
+ }
+
@Test
public void consistencyCheckInit() throws Exception {
DataStoreState state = setUp();