You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by am...@apache.org on 2019/03/15 14:24:30 UTC
svn commit: r1855603 - in /jackrabbit/oak/branches/1.10: ./
oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/
oak-it/src/test/java/org/apach...
Author: amitj
Date: Fri Mar 15 14:24:29 2019
New Revision: 1855603
URL: http://svn.apache.org/viewvc?rev=1855603&view=rev
Log:
OAK-8115: Allow disabling of BlobTracker when not required
- Merge r1855478 from trunk
Modified:
jackrabbit/oak/branches/1.10/ (props changed)
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java
jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
jackrabbit/oak/branches/1.10/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
jackrabbit/oak/branches/1.10/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
Propchange: jackrabbit/oak/branches/1.10/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Fri Mar 15 14:24:29 2019
@@ -1,3 +1,3 @@
/jackrabbit/oak/branches/1.0:1665962
-/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854701,1854827,1854848,1854859,1854930,1854990-1854991,1855221,1855477
+/jackrabbit/oak/trunk:1850874,1850882,1851236,1851253,1851451,1851533-1851535,1851619,1852052,1852084,1852120,1852451,1852492-1852493,1852528,1852582,1852584,1852601,1852920,1853141,1853229,1853393,1853429,1853433,1853441,1853866,1853868,1853870,1853893,1853969,1853997,1854034,1854044,1854058,1854113,1854373,1854377,1854380,1854385,1854401,1854403,1854455,1854461-1854462,1854466,1854468,1854515,1854533,1854701,1854827,1854848,1854859,1854930,1854990-1854991,1855221,1855477-1855478
/jackrabbit/trunk:1345480
Modified: jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java (original)
+++ jackrabbit/oak/branches/1.10/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTracker.java Fri Mar 15 14:24:29 2019
@@ -23,7 +23,6 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ScheduledExecutorService;
@@ -53,6 +52,7 @@ import static com.google.common.io.Files
import static com.google.common.io.Files.newWriter;
import static java.io.File.createTempFile;
import static java.lang.System.currentTimeMillis;
+import static java.util.Collections.emptyIterator;
import static java.util.Collections.synchronizedList;
import static java.util.UUID.randomUUID;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
@@ -83,41 +83,35 @@ import static org.apache.jackrabbit.oak.
public class BlobIdTracker implements Closeable, BlobTracker {
private static final Logger LOG = LoggerFactory.getLogger(BlobIdTracker.class);
- /**
- * System property to skip tracker. If set will skip:
- * * Snapshots (No-op)
- * * Retrieve (return empty)
- * * Add (No-op)
- */
- private final boolean SKIP_TRACKER = Boolean.getBoolean("oak.datastore.skipTracker");
-
private static final String datastoreMeta = "blobids";
private static final String fileNamePrefix = "blob";
private static final String mergedFileSuffix = ".refs";
private static final String snapshotMarkerSuffix = ".snapshot";
/* Local instance identifier */
- private final String instanceId = randomUUID().toString();
+ private String instanceId = randomUUID().toString();
- private final SharedDataStore datastore;
- private final long snapshotInterval;
- private final ActiveDeletionTracker deleteTracker;
+ private SharedDataStore datastore;
+ private long snapshotInterval;
+ private ActiveDeletionTracker deleteTracker;
protected BlobIdStore store;
- private final ScheduledExecutorService scheduler;
+ private ScheduledExecutorService scheduler;
private String prefix;
private File rootDir;
- public BlobIdTracker(String path, String repositoryId, long snapshotIntervalSecs, SharedDataStore datastore)
+ private BlobIdTracker() {}
+
+ private BlobIdTracker(String path, String repositoryId, long snapshotIntervalSecs, SharedDataStore datastore)
throws IOException {
this(path, repositoryId, newSingleThreadScheduledExecutor(), snapshotIntervalSecs, snapshotIntervalSecs,
datastore);
}
- public BlobIdTracker(String path, String repositoryId, ScheduledExecutorService scheduler, long snapshotDelaySecs,
+ private BlobIdTracker(String path, String repositoryId, ScheduledExecutorService scheduler, long snapshotDelaySecs,
long snapshotIntervalSecs, SharedDataStore datastore) throws IOException {
String root = concat(path, datastoreMeta);
this.rootDir = new File(root);
@@ -138,6 +132,66 @@ public class BlobIdTracker implements Cl
}
}
+ public static BlobIdTracker build(String path, String repositoryId, long snapshotIntervalSecs, SharedDataStore datastore)
+ throws IOException {
+ if (snapshotIntervalSecs > 0) {
+ return new BlobIdTracker(path, repositoryId, snapshotIntervalSecs, datastore);
+ }
+ return new BlobIdTracker() {
+ public ActiveDeletionTracker getDeleteTracker() {
+ return new ActiveDeletionTracker() {
+ @Override
+ public void track(File recs) {
+ }
+
+ @Override
+ public File retrieve(String path) {
+ File copiedRecsFile = new File(path);
+ return copiedRecsFile;
+ }
+
+ @Override
+ public void reconcile(File recs) {
+ }
+
+ @Override
+ public Iterator<String> filter(File recs) {
+ return emptyIterator();
+ }
+ };
+ }
+
+ @Override public void remove(File recs, Options options) {
+ }
+
+ @Override public void remove(File recs) {
+ }
+
+ @Override public void remove(Iterator<String> recs) {
+ }
+
+ @Override public void add(String id) {
+ }
+
+ @Override public void add(Iterator<String> recs) {
+ }
+
+ @Override public void add(File recs) {
+ }
+
+ @Override public Iterator<String> get() {
+ return emptyIterator();
+ }
+
+ @Override public File get(String path) {
+ return new File(path);
+ }
+
+ @Override public void close() {
+ }
+ };
+ }
+
public ActiveDeletionTracker getDeleteTracker() {
return deleteTracker;
}
@@ -164,21 +218,15 @@ public class BlobIdTracker implements Cl
}
@Override public void add(String id) throws IOException {
- if (!SKIP_TRACKER) {
- store.addRecord(id);
- }
+ store.addRecord(id);
}
@Override public void add(Iterator<String> recs) throws IOException {
- if (!SKIP_TRACKER) {
- store.addRecords(recs);
- }
+ store.addRecords(recs);
}
@Override public void add(File recs) throws IOException {
- if (!SKIP_TRACKER) {
- store.addRecords(recs);
- }
+ store.addRecords(recs);
}
/**
@@ -194,11 +242,8 @@ public class BlobIdTracker implements Cl
*/
@Override public Iterator<String> get() throws IOException {
try {
- if (!SKIP_TRACKER) {
- globalMerge();
- return store.getRecords();
- }
- return Collections.emptyIterator();
+ globalMerge();
+ return store.getRecords();
} catch (IOException e) {
LOG.error("Error in retrieving blob records iterator", e);
throw e;
@@ -206,11 +251,8 @@ public class BlobIdTracker implements Cl
}
@Override public File get(String path) throws IOException {
- if (!SKIP_TRACKER) {
- globalMerge();
- return store.getRecords(path);
- }
- return new File(path);
+ globalMerge();
+ return store.getRecords(path);
}
/**
@@ -273,29 +315,27 @@ public class BlobIdTracker implements Cl
*/
private void snapshot(boolean skipStoreSnapshot) throws IOException {
try {
- if (!SKIP_TRACKER) {
- Stopwatch watch = Stopwatch.createStarted();
+ Stopwatch watch = Stopwatch.createStarted();
- if (!skipStoreSnapshot) {
- store.snapshot();
- LOG.debug("Completed snapshot in [{}]", watch.elapsed(TimeUnit.MILLISECONDS));
- }
+ if (!skipStoreSnapshot) {
+ store.snapshot();
+ LOG.debug("Completed snapshot in [{}]", watch.elapsed(TimeUnit.MILLISECONDS));
+ }
- watch = Stopwatch.createStarted();
- File recs = store.getBlobRecordsFile();
- datastore.addMetadataRecord(recs, (prefix + instanceId + System.currentTimeMillis() + mergedFileSuffix));
- LOG.info("Added blob id metadata record in DataStore in [{}]", watch.elapsed(TimeUnit.MILLISECONDS));
-
- try {
- forceDelete(recs);
- LOG.info("Deleted blob record file after snapshot and upload {}", recs);
-
- // Update the timestamp for the snapshot marker
- touch(getSnapshotMarkerFile());
- LOG.info("Updated snapshot marker");
- } catch (IOException e) {
- LOG.debug("Failed to in cleaning up {}", recs, e);
- }
+ watch = Stopwatch.createStarted();
+ File recs = store.getBlobRecordsFile();
+ datastore.addMetadataRecord(recs, (prefix + instanceId + System.currentTimeMillis() + mergedFileSuffix));
+ LOG.info("Added blob id metadata record in DataStore in [{}]", watch.elapsed(TimeUnit.MILLISECONDS));
+
+ try {
+ forceDelete(recs);
+ LOG.info("Deleted blob record file after snapshot and upload {}", recs);
+
+ // Update the timestamp for the snapshot marker
+ touch(getSnapshotMarkerFile());
+ LOG.info("Updated snapshot marker");
+ } catch (IOException e) {
+ LOG.debug("Failed to in cleaning up {}", recs, e);
}
} catch (Exception e) {
LOG.error("Error taking snapshot", e);
@@ -331,7 +371,7 @@ public class BlobIdTracker implements Cl
public static final String DELIM = ",";
/* Lock for operations on the active deletions file */
- private final ReentrantLock lock;
+ private ReentrantLock lock;
private static final Function<String, String> transformer = new Function<String, String>() {
@Nullable
@@ -349,6 +389,8 @@ public class BlobIdTracker implements Cl
lock = new ReentrantLock();
}
+ private ActiveDeletionTracker() {}
+
/**
* Adds the ids in the file provided to the tracked deletions.
* @param recs the deleted ids to track
Modified: jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerClusterSharedTest.java Fri Mar 15 14:24:29 2019
@@ -167,7 +167,7 @@ public class BlobIdTrackerClusterSharedT
Cluster(String repoId, String path, TemporaryFolder folder) throws Exception {
this.dataStore = getBlobStore(root);
- this.tracker = new BlobIdTracker(path, repoId, 100 * 60, dataStore);
+ this.tracker = BlobIdTracker.build(path, repoId, 100 * 60, dataStore);
this.scheduler = newSingleThreadScheduledExecutor();
this.folder = folder;
}
Modified: jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerStoreTest.java Fri Mar 15 14:24:29 2019
@@ -87,7 +87,7 @@ public class BlobIdTrackerStoreTest {
}
private BlobIdTracker initTracker() throws IOException {
- return new BlobIdTracker(root.getAbsolutePath(),
+ return BlobIdTracker.build(root.getAbsolutePath(),
repoId, 5 * 60, dataStore);
}
Modified: jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/BlobIdTrackerTest.java Fri Mar 15 14:24:29 2019
@@ -96,7 +96,7 @@ public class BlobIdTrackerTest {
dataStore = getBlobStore(folder.newFolder());
}
this.repoId = randomUUID().toString();
- this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
+ this.tracker = BlobIdTracker.build(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
this.scheduler = newSingleThreadScheduledExecutor();
closer.register(tracker);
closer.register(new ExecutorCloser(scheduler));
@@ -206,11 +206,9 @@ public class BlobIdTrackerTest {
public void snapshotRetrieveIgnored() throws Exception {
LOG.info("In snapshotRetrieveIgnored");
- System.setProperty("oak.datastore.skipTracker", "true");
-
// Close and open a new object to use the system property
closer.close();
- this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
+ this.tracker = BlobIdTracker.build(root.getAbsolutePath(), repoId, 0, dataStore);
this.scheduler = newSingleThreadScheduledExecutor();
closer.register(tracker);
closer.register(new ExecutorCloser(scheduler));
@@ -220,7 +218,6 @@ public class BlobIdTrackerTest {
ScheduledFuture<?> scheduledFuture =
scheduler.schedule(tracker.new SnapshotJob(), 0, TimeUnit.MILLISECONDS);
scheduledFuture.get();
- assertEquals("References file not empty", 0, tracker.store.getBlobRecordsFile().length());
Set<String> retrieved = retrieveFile(tracker, folder);
assertTrue(retrieved.isEmpty());
@@ -229,7 +226,6 @@ public class BlobIdTrackerTest {
assertTrue(retrieved.isEmpty());
} finally {
//reset the skip tracker system prop
- System.clearProperty("oak.datastore.skipTracker");
}
}
@@ -249,7 +245,7 @@ public class BlobIdTrackerTest {
List<String> offlineLoad = range(0, 1000);
FileIOUtils.writeStrings(offlineLoad.iterator(), offline, false);
- this.tracker = new BlobIdTracker(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
+ this.tracker = BlobIdTracker.build(root.getAbsolutePath(), repoId, 100 * 60, dataStore);
this.scheduler = newSingleThreadScheduledExecutor();
closer.register(tracker);
closer.register(new ExecutorCloser(scheduler));
Modified: jackrabbit/oak/branches/1.10/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-it/src/test/java/org/apache/jackrabbit/oak/plugins/blob/datastore/DataStoreTrackerGCTest.java Fri Mar 15 14:24:29 2019
@@ -638,7 +638,7 @@ public class DataStoreTrackerGCTest {
REPOSITORY.getNameFromId(repoId));
String trackerRoot = folder.newFolder(clusterName).getAbsolutePath();
- tracker = new BlobIdTracker(trackerRoot,
+ tracker = BlobIdTracker.build(trackerRoot,
repoId, 86400, (SharedDataStore) blobStore);
// add the tracker to the blobStore
((BlobTrackingStore) blobStore).addTracker(tracker);
Modified: jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java (original)
+++ jackrabbit/oak/branches/1.10/oak-lucene/src/test/java/org/apache/jackrabbit/oak/plugins/index/lucene/directory/ActiveDeletedBlobSyncTrackerTest.java Fri Mar 15 14:24:29 2019
@@ -86,7 +86,7 @@ public class ActiveDeletedBlobSyncTracke
nodeStore = SegmentNodeStoreBuilders.builder(store).build();
BlobTrackingStore trackingStore = (BlobTrackingStore) blobStore;
trackingStore.addTracker(
- new BlobIdTracker(blobTrackerRoot.getRoot().getAbsolutePath(), getOrCreateId(nodeStore), 600,
+ BlobIdTracker.build(blobTrackerRoot.getRoot().getAbsolutePath(), getOrCreateId(nodeStore), 600,
dsbs));
// set the blob store to skip writing blobs through the node store
editorProvider.setBlobStore(blobStore);
Modified: jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java (original)
+++ jackrabbit/oak/branches/1.10/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreRegistrar.java Fri Mar 15 14:24:29 2019
@@ -416,7 +416,7 @@ class SegmentNodeStoreRegistrar {
if (trackingStore.getTracker() != null) {
trackingStore.getTracker().close();
}
- trackingStore.addTracker(new BlobIdTracker(cfg.getRepositoryHome(), getOrCreateId(segmentNodeStore), cfg.getBlobSnapshotInterval(), sharedDataStore));
+ trackingStore.addTracker(BlobIdTracker.build(cfg.getRepositoryHome(), getOrCreateId(segmentNodeStore), cfg.getBlobSnapshotInterval(), sharedDataStore));
}
}
Modified: jackrabbit/oak/branches/1.10/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.10/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java?rev=1855603&r1=1855602&r2=1855603&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.10/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java (original)
+++ jackrabbit/oak/branches/1.10/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStoreService.java Fri Mar 15 14:24:29 2019
@@ -355,7 +355,7 @@ public class DocumentNodeStoreService {
trackingStore.getTracker().close();
}
((BlobTrackingStore) blobStore).addTracker(
- new BlobIdTracker(getRepositoryHome(), repoId,
+ BlobIdTracker.build(getRepositoryHome(), repoId,
config.blobTrackSnapshotIntervalInSecs(),
(SharedDataStore) blobStore));
}