You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2016/07/20 03:57:23 UTC
svn commit: r1753434 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
Author: amitj
Date: Wed Jul 20 03:57:23 2016
New Revision: 1753434
URL: http://svn.apache.org/viewvc?rev=1753434&view=rev
Log:
OAK-4200: [BlobGC] Improve collection times of blobs available
* System flag to ignore tracker so will download all blob ids from the data store
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1753434&r1=1753433&r2=1753434&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java Wed Jul 20 03:57:23 2016
@@ -30,6 +30,7 @@ import java.util.concurrent.locks.Reentr
import com.google.common.base.Function;
import com.google.common.base.Predicate;
+import com.google.common.collect.Iterators;
import com.google.common.collect.Lists;
import com.google.common.io.Files;
import org.apache.jackrabbit.core.data.DataRecord;
@@ -81,7 +82,14 @@ import static org.apache.jackrabbit.oak.
public class BlobIdTracker implements Closeable, BlobTracker {
private static final Logger LOG = LoggerFactory.getLogger(BlobIdTracker.class);
- private static final String datastoreMeta = "blobreferences";
+ /**
+ * System property to skip tracker. If set will skip:
+ * * Snapshots (No-op)
+ * * Retrieve (return empty)
+ */
+ private final boolean SKIP_TRACKER = Boolean.getBoolean("oak.datastore.skipTracker");
+
+ private static final String datastoreMeta = "blobids";
private static final String fileNamePrefix = "blob";
private static final String mergedFileSuffix = ".refs";
@@ -162,8 +170,11 @@ public class BlobIdTracker implements Cl
@Override
public Iterator<String> get() throws IOException {
try {
- globalMerge();
- return store.getRecords();
+ if (!SKIP_TRACKER) {
+ globalMerge();
+ return store.getRecords();
+ }
+ return Iterators.emptyIterator();
} catch (IOException e) {
LOG.error("Error in retrieving blob records iterator", e);
throw e;
@@ -172,8 +183,11 @@ public class BlobIdTracker implements Cl
@Override
public File get(String path) throws IOException {
- globalMerge();
- return store.getRecords(path);
+ if (!SKIP_TRACKER) {
+ globalMerge();
+ return store.getRecords(path);
+ }
+ return new File(path);
}
private void globalMerge() throws IOException {
@@ -225,12 +239,12 @@ public class BlobIdTracker implements Cl
private void snapshot() throws IOException {
InputStream inputStream = null;
try {
- store.snapshot();
+ if (!SKIP_TRACKER) {
+ store.snapshot();
- inputStream =
- asByteSource(store.getBlobRecordsFile()).openBufferedStream();
- datastore.addMetadataRecord(inputStream,
- (prefix + instanceId + mergedFileSuffix));
+ inputStream = asByteSource(store.getBlobRecordsFile()).openBufferedStream();
+ datastore.addMetadataRecord(inputStream, (prefix + instanceId + mergedFileSuffix));
+ }
} catch (Exception e) {
LOG.error("Error taking snapshot", e);
throw new IOException("Snapshot error", e);
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java?rev=1753434&r1=1753433&r2=1753434&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java Wed Jul 20 03:57:23 2016
@@ -20,6 +20,7 @@ package org.apache.jackrabbit.oak.plugin
import java.io.Closeable;
import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
@@ -123,6 +124,32 @@ public class BlobIdTrackerTest {
assertEquals("Extra elements after remove", initAdd, retrieve(tracker));
}
+ @Test
+ public void snapshotRetrieveIgnored() throws Exception {
+ System.setProperty("oak.datastore.skipTracker", "true");
+ this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
+ this.scheduler = newSingleThreadScheduledExecutor();
+
+ try {
+ Set<String> initAdd = add(tracker, range(0, 10000));
+ ScheduledFuture<?> scheduledFuture =
+ scheduler.schedule(tracker.new SnapshotJob(), 0, TimeUnit.MILLISECONDS);
+ scheduledFuture.get();
+ assertEquals("References file not empty", 0, tracker.store.getBlobRecordsFile().length());
+
+ Set<String> retrieved = retrieveFile(tracker, folder);
+ assertTrue(retrieved.isEmpty());
+
+ retrieved = retrieve(tracker);
+ assertTrue(retrieved.isEmpty());
+ } finally {
+ //reset the skip tracker system prop
+ System.clearProperty("oak.datastore.skipTracker");
+ this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
+ this.scheduler = newSingleThreadScheduledExecutor();
+ }
+ }
+
private static Set<String> read(List<DataRecord> recs)
throws IOException, DataStoreException {
Set<String> ids = newHashSet();
@@ -153,6 +180,13 @@ public class BlobIdTrackerTest {
return retrieved;
}
+ private static Set<String> retrieveFile(BlobIdTracker tracker, TemporaryFolder folder) throws IOException {
+ File f = folder.newFile();
+ Set<String> retrieved = readStringsAsSet(
+ new FileInputStream(tracker.get(f.getAbsolutePath())), false);
+ return retrieved;
+ }
+
private static void remove(BlobTracker tracker, File temp, Set<String> initAdd,
List<String> ints) throws IOException {
writeStrings(ints.iterator(), temp, false);