You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2018/04/24 15:19:15 UTC
svn commit: r1830010 -
/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/
Author: frm
Date: Tue Apr 24 15:19:15 2018
New Revision: 1830010
URL: http://svn.apache.org/viewvc?rev=1830010&view=rev
Log:
OAK-7445 - Introduce a cleanup-first compaction strategy
Introduce a new garbage collection strategy that runs a pre-cleanup phase
before the compaction phase. The new strategy doesn't replace the default
garbage collection strategy, but lives alongside it hidden by a feature flag.
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java (with props)
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java?rev=1830010&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.ESTIMATION;
+import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.IDLE;
+
+import java.io.IOException;
+import java.util.List;
+
+import com.google.common.base.Predicate;
+import org.apache.jackrabbit.oak.segment.Revisions;
+import org.apache.jackrabbit.oak.segment.SegmentCache;
+import org.apache.jackrabbit.oak.segment.SegmentReader;
+import org.apache.jackrabbit.oak.segment.SegmentTracker;
+import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
+import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+
+abstract class AbstractGarbageCollectionStrategy implements GarbageCollectionStrategy {
+
+ abstract EstimationStrategy getFullEstimationStrategy();
+
+ abstract EstimationStrategy getTailEstimationStrategy();
+
+ abstract CompactionStrategy getFullCompactionStrategy();
+
+ abstract CompactionStrategy getTailCompactionStrategy();
+
+ abstract CleanupStrategy getCleanupStrategy();
+
+ @Override
+ public void collectGarbage(Context context) throws IOException {
+ switch (context.getGCOptions().getGCType()) {
+ case FULL:
+ collectFullGarbage(context);
+ break;
+ case TAIL:
+ collectTailGarbage(context);
+ break;
+ default:
+ throw new IllegalStateException("Invalid GC type");
+ }
+ }
+
+ @Override
+ public void collectFullGarbage(Context context) throws IOException {
+ run(context, getFullEstimationStrategy(), getFullCompactionStrategy());
+ }
+
+ @Override
+ public void collectTailGarbage(Context context) throws IOException {
+ run(context, getTailEstimationStrategy(), getTailCompactionStrategy());
+ }
+
+ @Override
+ public CompactionResult compactFull(Context context) throws IOException {
+ return getFullCompactionStrategy().compact(newCompactionStrategyContext(context));
+ }
+
+ @Override
+ public CompactionResult compactTail(Context context) throws IOException {
+ return getTailCompactionStrategy().compact(newCompactionStrategyContext(context));
+ }
+
+ @Override
+ public List<String> cleanup(Context context) throws IOException {
+ return cleanup(context, CompactionResult.skipped(
+ context.getLastCompactionType(),
+ getGcGeneration(context),
+ context.getGCOptions(),
+ context.getRevisions().getHead(),
+ context.getGCCount()
+ ));
+ }
+
+ void run(Context context, EstimationStrategy estimationStrategy, CompactionStrategy compactionStrategy) throws IOException {
+ try {
+ context.getGCListener().info("started");
+
+ long dt = System.currentTimeMillis() - context.getLastSuccessfulGC();
+
+ if (dt < context.getGCBackOff()) {
+ context.getGCListener().skipped("skipping garbage collection as it already ran less than {} hours ago ({} s).", context.getGCBackOff() / 3600000, dt / 1000);
+ return;
+ }
+
+ boolean sufficientEstimatedGain = true;
+ if (context.getGCOptions().isEstimationDisabled()) {
+ context.getGCListener().info("estimation skipped because it was explicitly disabled");
+ } else if (context.getGCOptions().isPaused()) {
+ context.getGCListener().info("estimation skipped because compaction is paused");
+ } else {
+ context.getGCListener().info("estimation started");
+ context.getGCListener().updateStatus(ESTIMATION.message());
+
+ PrintableStopwatch watch = PrintableStopwatch.createStarted();
+ EstimationResult estimation = estimationStrategy.estimate(newEstimationStrategyContext(context));
+ sufficientEstimatedGain = estimation.isGcNeeded();
+ String gcLog = estimation.getGcLog();
+ if (sufficientEstimatedGain) {
+ context.getGCListener().info("estimation completed in {}. {}", watch, gcLog);
+ } else {
+ context.getGCListener().skipped("estimation completed in {}. {}", watch, gcLog);
+ }
+ }
+
+ if (sufficientEstimatedGain) {
+ try (GCMemoryBarrier ignored = new GCMemoryBarrier(context.getSufficientMemory(), context.getGCListener(), context.getGCOptions())) {
+ if (context.getGCOptions().isPaused()) {
+ context.getGCListener().skipped("compaction paused");
+ } else if (!context.getSufficientMemory().get()) {
+ context.getGCListener().skipped("compaction skipped. Not enough memory");
+ } else {
+ CompactionResult compactionResult = compactionStrategy.compact(newCompactionStrategyContext(context));
+ if (compactionResult.isSuccess()) {
+ context.getSuccessfulGarbageCollectionListener().onSuccessfulGarbageCollection();
+ } else {
+ context.getGCListener().info("cleaning up after failed compaction");
+ }
+ context.getFileReaper().add(cleanup(context, compactionResult));
+ }
+ }
+ }
+ } finally {
+ context.getCompactionMonitor().finished();
+ context.getGCListener().updateStatus(IDLE.message());
+ }
+ }
+
+ private GCGeneration getGcGeneration(Context context) {
+ return context.getRevisions().getHead().getSegmentId().getGcGeneration();
+ }
+
+ private List<String> cleanup(Context context, CompactionResult compactionResult) throws IOException {
+ return getCleanupStrategy().cleanup(newCleanupStrategyContext(context, compactionResult));
+ }
+
+ private EstimationStrategy.Context newEstimationStrategyContext(Context context) {
+ return new EstimationStrategy.Context() {
+
+ @Override
+ public long getSizeDelta() {
+ return context.getGCOptions().getGcSizeDeltaEstimation();
+ }
+
+ @Override
+ public long getCurrentSize() {
+ return context.getTarFiles().size();
+ }
+
+ @Override
+ public GCJournal getGCJournal() {
+ return context.getGCJournal();
+ }
+
+ };
+ }
+
+ private static CompactionStrategy.Context newCompactionStrategyContext(Context context) {
+ return new CompactionStrategy.Context() {
+
+ @Override
+ public SegmentTracker getSegmentTracker() {
+ return context.getSegmentTracker();
+ }
+
+ @Override
+ public GCListener getGCListener() {
+ return context.getGCListener();
+ }
+
+ @Override
+ public GCJournal getGCJournal() {
+ return context.getGCJournal();
+ }
+
+ @Override
+ public SegmentGCOptions getGCOptions() {
+ return context.getGCOptions();
+ }
+
+ @Override
+ public GCNodeWriteMonitor getCompactionMonitor() {
+ return context.getCompactionMonitor();
+ }
+
+ @Override
+ public SegmentReader getSegmentReader() {
+ return context.getSegmentReader();
+ }
+
+ @Override
+ public SegmentWriterFactory getSegmentWriterFactory() {
+ return context.getSegmentWriterFactory();
+ }
+
+ @Override
+ public Revisions getRevisions() {
+ return context.getRevisions();
+ }
+
+ @Override
+ public TarFiles getTarFiles() {
+ return context.getTarFiles();
+ }
+
+ @Override
+ public BlobStore getBlobStore() {
+ return context.getBlobStore();
+ }
+
+ @Override
+ public CancelCompactionSupplier getCanceller() {
+ return context.getCanceller();
+ }
+
+ @Override
+ public int getGCCount() {
+ return context.getGCCount();
+ }
+
+ @Override
+ public SuccessfulCompactionListener getSuccessfulCompactionListener() {
+ return context.getSuccessfulCompactionListener();
+ }
+
+ @Override
+ public Flusher getFlusher() {
+ return context.getFlusher();
+ }
+
+ };
+ }
+
+ private CleanupStrategy.Context newCleanupStrategyContext(Context context, CompactionResult compactionResult) {
+ return new CleanupStrategy.Context() {
+
+ @Override
+ public GCListener getGCListener() {
+ return context.getGCListener();
+ }
+
+ @Override
+ public SegmentCache getSegmentCache() {
+ return context.getSegmentCache();
+ }
+
+ @Override
+ public SegmentTracker getSegmentTracker() {
+ return context.getSegmentTracker();
+ }
+
+ @Override
+ public FileStoreStats getFileStoreStats() {
+ return context.getFileStoreStats();
+ }
+
+ @Override
+ public GCNodeWriteMonitor getCompactionMonitor() {
+ return context.getCompactionMonitor();
+ }
+
+ @Override
+ public GCJournal getGCJournal() {
+ return context.getGCJournal();
+ }
+
+ @Override
+ public Predicate<GCGeneration> getReclaimer() {
+ return compactionResult.reclaimer();
+ }
+
+ @Override
+ public TarFiles getTarFiles() {
+ return context.getTarFiles();
+ }
+
+ @Override
+ public Revisions getRevisions() {
+ return context.getRevisions();
+ }
+
+ @Override
+ public String getCompactedRootId() {
+ return compactionResult.getCompactedRootId().toString10();
+ }
+
+ @Override
+ public String getSegmentEvictionReason() {
+ return compactionResult.gcInfo();
+ }
+
+ };
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractGarbageCollectionStrategy.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java?rev=1830010&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java Tue Apr 24 15:19:15 2018
@@ -0,0 +1,204 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.oak.segment.SegmentId.isDataSegmentId;
+import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.CLEANUP;
+import static org.apache.jackrabbit.oak.segment.file.PrintableBytes.newPrintableBytes;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+
+import javax.annotation.Nonnull;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Predicate;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+import org.apache.jackrabbit.oak.segment.file.tar.CleanupContext;
+import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
+
+class CleanupFirstCompactionStrategy implements CompactionStrategy {
+
+ private final GarbageCollectionStrategy.Context parentContext;
+
+ private final CompactionStrategy strategy;
+
+ CleanupFirstCompactionStrategy(GarbageCollectionStrategy.Context parentContext, CompactionStrategy strategy) {
+ this.parentContext = parentContext;
+ this.strategy = strategy;
+ }
+
+ @Override
+ public CompactionResult compact(Context context) throws IOException {
+
+ // This is a slightly modified version of the default cleanup phase when
+ // invoked with a successful compaction result. There are some important
+ // differences deriving from the fact that we are assuming that the compaction
+ // for `newGeneration` is going to succeed.
+
+ // First, we don't have a RecordId for the compacted state, because it didn't
+ // happen yet. This shouldn't matter, because we are not going to advance the
+ // `gcJournal` to the `RecordId` of the compacted state.
+
+ // Second, we are using a custom reclaimer that is similar to the one returned
+ // by `newOldReclaimer`, but that also takes in consideration that
+ // `newGeneration` has not been committed yet, and the most recent transient
+ // state shouldn't be removed.
+
+ // Third, we don't clear the segment cache. There might be transient segments
+ // in there, and we don't want those segments to be removed.
+
+ // Fourth, the following code assumes the number of retained generations fixed
+ // to two, which is also the default value for the Segment Store. A complete
+ // solution should be flexible enough to accommodate other values for the
+ // number of retained generations.
+
+ PrintableStopwatch watch = PrintableStopwatch.createStarted();
+
+ Predicate<GCGeneration> reclaimer;
+
+ GCGeneration currentGeneration = context.getRevisions().getHead().getSegmentId().getGcGeneration();
+
+ switch (context.getGCOptions().getGCType()) {
+ case FULL:
+ reclaimer = generation -> {
+ if (generation == null) {
+ return false;
+ }
+ if (generation.getFullGeneration() < currentGeneration.getFullGeneration()) {
+ return true;
+ }
+ if (generation.getFullGeneration() > currentGeneration.getFullGeneration()) {
+ return true;
+ }
+ return generation.getGeneration() < currentGeneration.getGeneration() && !generation.isCompacted();
+ };
+ break;
+ case TAIL:
+ reclaimer = generation -> {
+ if (generation == null) {
+ return false;
+ }
+ if (generation.getFullGeneration() < currentGeneration.getFullGeneration() - 1) {
+ return true;
+ }
+ if (generation.getFullGeneration() == currentGeneration.getFullGeneration() - 1) {
+ return !generation.isCompacted();
+ }
+ if (generation.getFullGeneration() > currentGeneration.getFullGeneration()) {
+ return true;
+ }
+ return generation.getGeneration() < currentGeneration.getGeneration() && !generation.isCompacted();
+ };
+ break;
+ default:
+ throw new IllegalArgumentException("invalid garbage collection type");
+ }
+
+ context.getGCListener().info("pre-compaction cleanup started");
+ context.getGCListener().updateStatus(CLEANUP.message());
+
+ // Suggest to the JVM that now would be a good time to clear stale weak
+ // references in the SegmentTracker
+
+ System.gc();
+
+ TarFiles.CleanupResult cleanupResult = context.getTarFiles().cleanup(newCleanupContext(context, reclaimer));
+
+ if (cleanupResult.isInterrupted()) {
+ context.getGCListener().info("cleanup interrupted");
+ }
+
+ context.getSegmentTracker().clearSegmentIdTables(cleanupResult.getReclaimedSegmentIds(), "[pre-compaction cleanup]");
+ context.getGCListener().info("cleanup marking files for deletion: {}", toFileNames(cleanupResult.getRemovableFiles()));
+
+ long finalSize = context.getTarFiles().size();
+ long reclaimedSize = cleanupResult.getReclaimedSize();
+ parentContext.getFileStoreStats().reclaimed(reclaimedSize);
+ context.getGCListener().cleaned(reclaimedSize, finalSize);
+ context.getGCListener().info(
+ "pre-compaction cleanup completed in {}. Post cleanup size is {} and space reclaimed {}.",
+ watch,
+ newPrintableBytes(finalSize),
+ newPrintableBytes(reclaimedSize));
+ parentContext.getFileReaper().add(cleanupResult.getRemovableFiles());
+
+ // Invoke the wrapped compaction phase
+
+ return strategy.compact(context);
+ }
+
+ private static CleanupContext newCleanupContext(Context context, Predicate<GCGeneration> old) {
+ return new CleanupContext() {
+
+ private boolean isUnreferencedBulkSegment(UUID id, boolean referenced) {
+ return !isDataSegmentId(id.getLeastSignificantBits()) && !referenced;
+ }
+
+ private boolean isOldDataSegment(UUID id, GCGeneration generation) {
+ return isDataSegmentId(id.getLeastSignificantBits()) && old.apply(generation);
+ }
+
+ @Override
+ public Collection<UUID> initialReferences() {
+ Set<UUID> references = newHashSet();
+ for (SegmentId id : context.getSegmentTracker().getReferencedSegmentIds()) {
+ if (id.isBulkSegmentId()) {
+ references.add(id.asUUID());
+ }
+ }
+ return references;
+ }
+
+ @Override
+ public boolean shouldReclaim(UUID id, GCGeneration generation, boolean referenced) {
+ return isUnreferencedBulkSegment(id, referenced) || isOldDataSegment(id, generation);
+ }
+
+ @Override
+ public boolean shouldFollow(UUID from, UUID to) {
+ return !isDataSegmentId(to.getLeastSignificantBits());
+ }
+
+ };
+ }
+
+ private static String toFileNames(@Nonnull List<String> files) {
+ if (files.isEmpty()) {
+ return "none";
+ } else {
+ return Joiner.on(",").join(files);
+ }
+ }
+
+ private static GCGeneration getGcGeneration(Context context) {
+ return context.getRevisions().getHead().getSegmentId().getGcGeneration();
+ }
+
+ private static long size(Context context) {
+ return context.getTarFiles().size();
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstCompactionStrategy.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java?rev=1830010&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import java.io.IOException;
+
+class CleanupFirstGarbageCollectionStrategy extends AbstractGarbageCollectionStrategy {
+
+ @Override
+ EstimationStrategy getFullEstimationStrategy() {
+ return new FullSizeDeltaEstimationStrategy();
+ }
+
+ @Override
+ EstimationStrategy getTailEstimationStrategy() {
+ return new TailSizeDeltaEstimationStrategy();
+ }
+
+ @Override
+ CompactionStrategy getFullCompactionStrategy() {
+ return new FullCompactionStrategy();
+ }
+
+ @Override
+ CompactionStrategy getTailCompactionStrategy() {
+ return new FallbackCompactionStrategy(new TailCompactionStrategy(), new FullCompactionStrategy());
+ }
+
+ @Override
+ CleanupStrategy getCleanupStrategy() {
+ return new DefaultCleanupStrategy();
+ }
+
+ @Override
+ void run(Context context, EstimationStrategy estimationStrategy, CompactionStrategy compactionStrategy) throws IOException {
+ super.run(context, estimationStrategy, new CleanupFirstCompactionStrategy(context, compactionStrategy));
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CleanupFirstGarbageCollectionStrategy.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java?rev=1830010&r1=1830009&r2=1830010&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/CompactionStrategy.java Tue Apr 24 15:19:15 2018
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.file;
+import java.io.IOException;
+
import org.apache.jackrabbit.oak.segment.Revisions;
import org.apache.jackrabbit.oak.segment.SegmentReader;
import org.apache.jackrabbit.oak.segment.SegmentTracker;
@@ -60,6 +62,6 @@ interface CompactionStrategy {
}
- CompactionResult compact(Context context);
+ CompactionResult compact(Context context) throws IOException;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java?rev=1830010&r1=1830009&r2=1830010&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/DefaultGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018
@@ -19,311 +19,31 @@
package org.apache.jackrabbit.oak.segment.file;
-import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.ESTIMATION;
-import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCStatus.IDLE;
-
-import java.io.IOException;
-import java.util.List;
-
-import com.google.common.base.Predicate;
-import org.apache.jackrabbit.oak.segment.Revisions;
-import org.apache.jackrabbit.oak.segment.SegmentCache;
-import org.apache.jackrabbit.oak.segment.SegmentReader;
-import org.apache.jackrabbit.oak.segment.SegmentTracker;
-import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
-import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
-import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
-import org.apache.jackrabbit.oak.spi.blob.BlobStore;
-
-class DefaultGarbageCollectionStrategy implements GarbageCollectionStrategy {
-
- private final EstimationStrategy fullEstimationStrategy = new FullSizeDeltaEstimationStrategy();
-
- private final EstimationStrategy tailEstimationStrategy = new TailSizeDeltaEstimationStrategy();
-
- private final CompactionStrategy fullCompactionStrategy = new FullCompactionStrategy();
-
- private final CompactionStrategy tailCompactionStrategy = new FallbackCompactionStrategy(new TailCompactionStrategy(), fullCompactionStrategy);
-
- private final CleanupStrategy cleanupStrategy = new DefaultCleanupStrategy();
-
- private GCGeneration getGcGeneration(Context context) {
- return context.getRevisions().getHead().getSegmentId().getGcGeneration();
- }
+class DefaultGarbageCollectionStrategy extends AbstractGarbageCollectionStrategy {
@Override
- public synchronized void collectGarbage(Context context) throws IOException {
- switch (context.getGCOptions().getGCType()) {
- case FULL:
- collectFullGarbage(context);
- break;
- case TAIL:
- collectTailGarbage(context);
- break;
- default:
- throw new IllegalStateException("Invalid GC type");
- }
+ EstimationStrategy getFullEstimationStrategy() {
+ return new FullSizeDeltaEstimationStrategy();
}
@Override
- public synchronized void collectFullGarbage(Context context) throws IOException {
- run(context, true, this::compactFull);
+ EstimationStrategy getTailEstimationStrategy() {
+ return new TailSizeDeltaEstimationStrategy();
}
@Override
- public synchronized void collectTailGarbage(Context context) throws IOException {
- run(context, false, this::compactTail);
- }
-
- private interface Compactor {
-
- CompactionResult compact(Context contex) throws IOException;
-
- }
-
- private void run(Context context, boolean full, Compactor compact) throws IOException {
- try {
- context.getGCListener().info("started");
-
- long dt = System.currentTimeMillis() - context.getLastSuccessfulGC();
-
- if (dt < context.getGCBackOff()) {
- context.getGCListener().skipped("skipping garbage collection as it already ran less than {} hours ago ({} s).", context.getGCBackOff() / 3600000, dt / 1000);
- return;
- }
-
- boolean sufficientEstimatedGain = true;
- if (context.getGCOptions().isEstimationDisabled()) {
- context.getGCListener().info("estimation skipped because it was explicitly disabled");
- } else if (context.getGCOptions().isPaused()) {
- context.getGCListener().info("estimation skipped because compaction is paused");
- } else {
- context.getGCListener().info("estimation started");
- context.getGCListener().updateStatus(ESTIMATION.message());
-
- PrintableStopwatch watch = PrintableStopwatch.createStarted();
- EstimationResult estimation = estimateCompactionGain(context, full);
- sufficientEstimatedGain = estimation.isGcNeeded();
- String gcLog = estimation.getGcLog();
- if (sufficientEstimatedGain) {
- context.getGCListener().info("estimation completed in {}. {}", watch, gcLog);
- } else {
- context.getGCListener().skipped("estimation completed in {}. {}", watch, gcLog);
- }
- }
-
- if (sufficientEstimatedGain) {
- try (GCMemoryBarrier ignored = new GCMemoryBarrier(context.getSufficientMemory(), context.getGCListener(), context.getGCOptions())) {
- if (context.getGCOptions().isPaused()) {
- context.getGCListener().skipped("compaction paused");
- } else if (!context.getSufficientMemory().get()) {
- context.getGCListener().skipped("compaction skipped. Not enough memory");
- } else {
- CompactionResult compactionResult = compact.compact(context);
- if (compactionResult.isSuccess()) {
- context.getSuccessfulGarbageCollectionListener().onSuccessfulGarbageCollection();
- } else {
- context.getGCListener().info("cleaning up after failed compaction");
- }
- context.getFileReaper().add(cleanup(context, compactionResult));
- }
- }
- }
- } finally {
- context.getCompactionMonitor().finished();
- context.getGCListener().updateStatus(IDLE.message());
- }
- }
-
- private static CompactionStrategy.Context newCompactionStrategyContext(Context context) {
- return new CompactionStrategy.Context() {
-
- @Override
- public SegmentTracker getSegmentTracker() {
- return context.getSegmentTracker();
- }
-
- @Override
- public GCListener getGCListener() {
- return context.getGCListener();
- }
-
- @Override
- public GCJournal getGCJournal() {
- return context.getGCJournal();
- }
-
- @Override
- public SegmentGCOptions getGCOptions() {
- return context.getGCOptions();
- }
-
- @Override
- public GCNodeWriteMonitor getCompactionMonitor() {
- return context.getCompactionMonitor();
- }
-
- @Override
- public SegmentReader getSegmentReader() {
- return context.getSegmentReader();
- }
-
- @Override
- public SegmentWriterFactory getSegmentWriterFactory() {
- return context.getSegmentWriterFactory();
- }
-
- @Override
- public Revisions getRevisions() {
- return context.getRevisions();
- }
-
- @Override
- public TarFiles getTarFiles() {
- return context.getTarFiles();
- }
-
- @Override
- public BlobStore getBlobStore() {
- return context.getBlobStore();
- }
-
- @Override
- public CancelCompactionSupplier getCanceller() {
- return context.getCanceller();
- }
-
- @Override
- public int getGCCount() {
- return context.getGCCount();
- }
-
- @Override
- public SuccessfulCompactionListener getSuccessfulCompactionListener() {
- return context.getSuccessfulCompactionListener();
- }
-
- @Override
- public Flusher getFlusher() {
- return context.getFlusher();
- }
-
- };
- }
-
- @Override
- public synchronized CompactionResult compactFull(Context context) {
- return fullCompactionStrategy.compact(newCompactionStrategyContext(context));
+ CompactionStrategy getFullCompactionStrategy() {
+ return new FullCompactionStrategy();
}
@Override
- public synchronized CompactionResult compactTail(Context context) {
- return tailCompactionStrategy.compact(newCompactionStrategyContext(context));
- }
-
- private EstimationResult estimateCompactionGain(Context context, boolean full) {
- EstimationStrategy strategy;
-
- if (full) {
- strategy = fullEstimationStrategy;
- } else {
- strategy = tailEstimationStrategy;
- }
-
- return estimateCompactionGain(context, strategy);
- }
-
- private EstimationResult estimateCompactionGain(Context context, EstimationStrategy strategy) {
- return strategy.estimate(new EstimationStrategy.Context() {
-
- @Override
- public long getSizeDelta() {
- return context.getGCOptions().getGcSizeDeltaEstimation();
- }
-
- @Override
- public long getCurrentSize() {
- return context.getTarFiles().size();
- }
-
- @Override
- public GCJournal getGCJournal() {
- return context.getGCJournal();
- }
-
- });
+ CompactionStrategy getTailCompactionStrategy() {
+ return new FallbackCompactionStrategy(new TailCompactionStrategy(), new FullCompactionStrategy());
}
@Override
- public synchronized List<String> cleanup(Context context) throws IOException {
- return cleanup(context, CompactionResult.skipped(
- context.getLastCompactionType(),
- getGcGeneration(context),
- context.getGCOptions(),
- context.getRevisions().getHead(),
- context.getGCCount()
- ));
- }
-
- private List<String> cleanup(Context context, CompactionResult compactionResult) throws IOException {
- return cleanupStrategy.cleanup(new CleanupStrategy.Context() {
-
- @Override
- public GCListener getGCListener() {
- return context.getGCListener();
- }
-
- @Override
- public SegmentCache getSegmentCache() {
- return context.getSegmentCache();
- }
-
- @Override
- public SegmentTracker getSegmentTracker() {
- return context.getSegmentTracker();
- }
-
- @Override
- public FileStoreStats getFileStoreStats() {
- return context.getFileStoreStats();
- }
-
- @Override
- public GCNodeWriteMonitor getCompactionMonitor() {
- return context.getCompactionMonitor();
- }
-
- @Override
- public GCJournal getGCJournal() {
- return context.getGCJournal();
- }
-
- @Override
- public Predicate<GCGeneration> getReclaimer() {
- return compactionResult.reclaimer();
- }
-
- @Override
- public TarFiles getTarFiles() {
- return context.getTarFiles();
- }
-
- @Override
- public Revisions getRevisions() {
- return context.getRevisions();
- }
-
- @Override
- public String getCompactedRootId() {
- return compactionResult.getCompactedRootId().toString10();
- }
-
- @Override
- public String getSegmentEvictionReason() {
- return compactionResult.gcInfo();
- }
-
- });
+ CleanupStrategy getCleanupStrategy() {
+ return new DefaultCleanupStrategy();
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java?rev=1830010&r1=1830009&r2=1830010&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FallbackCompactionStrategy.java Tue Apr 24 15:19:15 2018
@@ -19,6 +19,8 @@
package org.apache.jackrabbit.oak.segment.file;
+import java.io.IOException;
+
class FallbackCompactionStrategy implements CompactionStrategy {
private final CompactionStrategy primary;
@@ -31,7 +33,7 @@ class FallbackCompactionStrategy impleme
}
@Override
- public CompactionResult compact(Context context) {
+ public CompactionResult compact(Context context) throws IOException {
CompactionResult result = primary.compact(context);
if (result.isNotApplicable()) {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1830010&r1=1830009&r2=1830010&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Apr 24 15:19:15 2018
@@ -108,7 +108,15 @@ public class FileStore extends AbstractF
@Nonnull
private final SegmentNotFoundExceptionListener snfeListener;
- private final GarbageCollectionStrategy garbageCollectionStrategy = new DefaultGarbageCollectionStrategy();
+ private final GarbageCollectionStrategy garbageCollectionStrategy;
+
+ {
+ if (Boolean.getBoolean("gc.cleanup.first")) {
+ garbageCollectionStrategy = new SynchronizedGarbageCollectionStrategy(new CleanupFirstGarbageCollectionStrategy());
+ } else {
+ garbageCollectionStrategy = new SynchronizedGarbageCollectionStrategy(new DefaultGarbageCollectionStrategy());
+ }
+ }
FileStore(final FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException {
super(builder);
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java?rev=1830010&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java Tue Apr 24 15:19:15 2018
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.file;
+
+import java.io.IOException;
+import java.util.List;
+
+class SynchronizedGarbageCollectionStrategy implements GarbageCollectionStrategy {
+
+ private final Object lock = new Object();
+
+ private final GarbageCollectionStrategy strategy;
+
+ SynchronizedGarbageCollectionStrategy(GarbageCollectionStrategy strategy) {
+ this.strategy = strategy;
+ }
+
+ @Override
+ public void collectGarbage(Context context) throws IOException {
+ synchronized (lock) {
+ strategy.collectGarbage(context);
+ }
+ }
+
+ @Override
+ public void collectFullGarbage(Context context) throws IOException {
+ synchronized (lock) {
+ strategy.collectFullGarbage(context);
+ }
+ }
+
+ @Override
+ public void collectTailGarbage(Context context) throws IOException {
+ synchronized (lock) {
+ strategy.collectTailGarbage(context);
+ }
+ }
+
+ @Override
+ public CompactionResult compactFull(Context context) throws IOException {
+ synchronized (lock) {
+ return strategy.compactFull(context);
+ }
+ }
+
+ @Override
+ public CompactionResult compactTail(Context context) throws IOException {
+ synchronized (lock) {
+ return strategy.compactTail(context);
+ }
+ }
+
+ @Override
+ public List<String> cleanup(Context context) throws IOException {
+ synchronized (lock) {
+ return strategy.cleanup(context);
+ }
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/SynchronizedGarbageCollectionStrategy.java
------------------------------------------------------------------------------
svn:eol-style = native