You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2017/06/27 15:48:12 UTC
svn commit: r1800074 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/backup/impl/
main/java/org/apache/jackrabbit/oak/segment/
main/java/org/apache/jackrabbit/oak/segment/file/
test/java/org/apache/jackrabbit/oak/seg...
Author: mduerig
Date: Tue Jun 27 15:48:11 2017
New Revision: 1800074
URL: http://svn.apache.org/viewvc?rev=1800074&view=rev
Log:
OAK-5790: Chronologically rebase checkpoints on top of each other during compaction
Correctly update the compaction monitor
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/OnlineCompactor.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreBackupImpl.java Tue Jun 27 15:48:11 2017
@@ -40,7 +40,6 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
-import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.tooling.BasicReadOnlyBlobStore;
import org.slf4j.Logger;
@@ -81,8 +80,7 @@ public class FileStoreBackupImpl impleme
backup.getSegmentIdProvider(),
backup.getBlobStore(),
new WriterCacheManager.Default(),
- bufferWriter,
- GCNodeWriteMonitor.EMPTY
+ bufferWriter
);
Compactor compactor = new Compactor(
backup.getReader(),
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/backup/impl/FileStoreRestoreImpl.java Tue Jun 27 15:48:11 2017
@@ -36,7 +36,6 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.WriterCacheManager;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.FileStore;
-import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.slf4j.Logger;
@@ -75,8 +74,7 @@ public class FileStoreRestoreImpl implem
store.getSegmentIdProvider(),
store.getBlobStore(),
new WriterCacheManager.Default(),
- bufferWriter,
- GCNodeWriteMonitor.EMPTY
+ bufferWriter
);
SegmentGCOptions gcOptions = defaultGCOptions().setOffline();
Compactor compactor = new Compactor(
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriter.java Tue Jun 27 15:48:11 2017
@@ -69,7 +69,6 @@ import org.apache.jackrabbit.oak.api.Pro
import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState;
import org.apache.jackrabbit.oak.segment.WriteOperationHandler.WriteOperation;
-import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.DefaultNodeStateDiff;
@@ -105,9 +104,6 @@ public class DefaultSegmentWriter implem
@Nonnull
private final WriteOperationHandler writeOperationHandler;
- @Nonnull
- private final GCNodeWriteMonitor compactionMonitor;
-
/**
* Create a new instance of a {@code SegmentWriter}. Note the thread safety
* properties pointed out in the class comment.
@@ -127,8 +123,7 @@ public class DefaultSegmentWriter implem
@Nonnull SegmentIdProvider idProvider,
@Nullable BlobStore blobStore,
@Nonnull WriterCacheManager cacheManager,
- @Nonnull WriteOperationHandler writeOperationHandler,
- GCNodeWriteMonitor compactionMonitor
+ @Nonnull WriteOperationHandler writeOperationHandler
) {
this.store = checkNotNull(store);
this.reader = checkNotNull(reader);
@@ -136,7 +131,6 @@ public class DefaultSegmentWriter implem
this.blobStore = blobStore;
this.cacheManager = checkNotNull(cacheManager);
this.writeOperationHandler = checkNotNull(writeOperationHandler);
- this.compactionMonitor = checkNotNull(compactionMonitor);
}
@Override
@@ -814,7 +808,6 @@ public class DefaultSegmentWriter implem
// generation (e.g. due to compaction). Put it into the cache for
// deduplication of hard links to it (e.g. checkpoints).
nodeCache.put(getStableId(stableIdBytes), recordId, cost(state));
- compactionMonitor.compacted();
}
return recordId;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/DefaultSegmentWriterBuilder.java Tue Jun 27 15:48:11 2017
@@ -27,7 +27,6 @@ import com.google.common.base.Supplier;
import com.google.common.base.Suppliers;
import org.apache.jackrabbit.oak.segment.WriterCacheManager.Empty;
import org.apache.jackrabbit.oak.segment.file.FileStore;
-import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.apache.jackrabbit.oak.segment.memory.MemoryStore;
@@ -62,8 +61,6 @@ public final class DefaultSegmentWriterB
@Nonnull
private WriterCacheManager cacheManager = new WriterCacheManager.Default();
- private GCNodeWriteMonitor compactionMonitor = GCNodeWriteMonitor.EMPTY;
-
private DefaultSegmentWriterBuilder(@Nonnull String name) {
this.name = checkNotNull(name);
}
@@ -143,11 +140,6 @@ public final class DefaultSegmentWriterB
return this;
}
- public DefaultSegmentWriterBuilder withCompactionMonitor(GCNodeWriteMonitor compactionMonitor) {
- this.compactionMonitor = compactionMonitor;
- return this;
- }
-
/**
* Build a {@code SegmentWriter} for a {@code FileStore}.
*/
@@ -159,8 +151,7 @@ public final class DefaultSegmentWriterB
store.getSegmentIdProvider(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled),
- compactionMonitor
+ createWriter(store, pooled)
);
}
@@ -188,8 +179,7 @@ public final class DefaultSegmentWriterB
public void flush(@Nonnull SegmentStore store) {
throw new UnsupportedOperationException("Cannot write to read-only store");
}
- },
- compactionMonitor
+ }
);
}
@@ -204,8 +194,7 @@ public final class DefaultSegmentWriterB
store.getSegmentIdProvider(),
store.getBlobStore(),
cacheManager,
- createWriter(store, pooled),
- compactionMonitor
+ createWriter(store, pooled)
);
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/OnlineCompactor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/OnlineCompactor.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/OnlineCompactor.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/OnlineCompactor.java Tue Jun 27 15:48:11 2017
@@ -73,22 +73,28 @@ public class OnlineCompactor {
@Nonnull
private final Supplier<Boolean> cancel;
+ @Nonnull
+ private final Runnable onNode;
+
/**
* Create a new instance based on the passed arguments.
* @param reader segment reader used to read from the segments
* @param writer segment writer used to serialise to segments
* @param blobStore the blob store or {@code null} if none
* @param cancel a flag that can be used to cancel the compaction process
+ * @param onNode notification call back for each compacted node
*/
public OnlineCompactor(
@Nonnull SegmentReader reader,
@Nonnull SegmentWriter writer,
@Nullable BlobStore blobStore,
- @Nonnull Supplier<Boolean> cancel) {
+ @Nonnull Supplier<Boolean> cancel,
+ @Nonnull Runnable onNode) {
this.writer = checkNotNull(writer);
this.reader = checkNotNull(reader);
this.blobStore = blobStore;
this.cancel = checkNotNull(cancel);
+ this.onNode = checkNotNull(onNode);
}
/**
@@ -165,6 +171,7 @@ public class OnlineCompactor {
NodeState nodeState = builder.getNodeState();
checkState(modCount == 0 || !(nodeState instanceof SegmentNodeState));
RecordId nodeId = writer.writeNode(nodeState, getStableIdBytes(after));
+ onNode.run();
return new SegmentNodeState(reader, writer, blobStore, nodeId);
} else {
return null;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Jun 27 15:48:11 2017
@@ -692,9 +692,9 @@ public class FileStore extends AbstractF
.withAccessTracking("COMPACT", statisticsProvider))
.withGeneration(newGeneration)
.withoutWriterPool()
- .withCompactionMonitor(compactionMonitor)
.build(FileStore.this);
- OnlineCompactor compactor = new OnlineCompactor(segmentReader, writer, getBlobStore(), cancel);
+ OnlineCompactor compactor = new OnlineCompactor(
+ segmentReader, writer, getBlobStore(), cancel, compactionMonitor::onNode);
SegmentNodeState after = compact(null, before, compactor, writer);
if (after == null) {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java Tue Jun 27 15:48:11 2017
@@ -60,7 +60,7 @@ public class GCNodeWriteMonitor {
this.gcProgressLog = gcProgressLog;
}
- public synchronized void compacted() {
+ public synchronized void onNode() {
nodes++;
if (gcProgressLog > 0 && nodes % gcProgressLog == 0) {
long ms = System.currentTimeMillis() - start;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java?rev=1800074&r1=1800073&r2=1800074&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/OnlineCompactorTest.java Tue Jun 27 15:48:11 2017
@@ -119,7 +119,7 @@ public class OnlineCompactorTest {
@Nonnull
private static OnlineCompactor createCompactor(FileStore fileStore, Supplier<Boolean> cancel) {
SegmentWriter writer = defaultSegmentWriterBuilder("c").withGeneration(1).build(fileStore);
- return new OnlineCompactor(fileStore.getReader(), writer, fileStore.getBlobStore(), cancel);
+ return new OnlineCompactor(fileStore.getReader(), writer, fileStore.getBlobStore(), cancel, () -> {});
}
private static void addNodes(SegmentNodeStore nodeStore, int count)