You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2016/12/05 12:38:04 UTC
svn commit: r1772656 - in /jackrabbit/oak/trunk: oak-doc/src/site/markdown/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/
oak-segment-tar/src/main/java/org/a...
Author: alexparvulescu
Date: Mon Dec 5 12:38:04 2016
New Revision: 1772656
URL: http://svn.apache.org/viewvc?rev=1772656&view=rev
Log:
OAK-5158 Online compaction progress monitor
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java (with props)
Modified:
jackrabbit/oak/trunk/oak-doc/src/site/markdown/osgi_config.md
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
Modified: jackrabbit/oak/trunk/oak-doc/src/site/markdown/osgi_config.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-doc/src/site/markdown/osgi_config.md?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-doc/src/site/markdown/osgi_config.md (original)
+++ jackrabbit/oak/trunk/oak-doc/src/site/markdown/osgi_config.md Mon Dec 5 12:38:04 2016
@@ -120,6 +120,9 @@ compaction.memoryThreshold (int) - 15
: The percentage of heap memory that should always be free while compaction runs.
If the available heap memory falls below the specified percentage, compaction will not be started or it will be aborted if it is already running.
+compaction.progressLog (long) - -1
+: Enables compaction progress logging at each set of compacted nodes. A value of `-1` disables the log.
+
standby (boolean) - false
: Determines if this Node Store is supposed to be used in standby mode.
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java Mon Dec 5 12:38:04 2016
@@ -26,6 +26,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.osgi.OsgiUtil.lookupConfigurationThenFramework;
import static org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener.IGNORE_SNFE;
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.FORCE_TIMEOUT_DEFAULT;
+import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.GC_PROGRESS_LOG_DEFAULT;
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.MEMORY_THRESHOLD_DEFAULT;
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.PAUSE_DEFAULT;
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.RETAINED_GENERATIONS_DEFAULT;
@@ -243,6 +244,13 @@ public class SegmentNodeStoreService ext
public static final String MEMORY_THRESHOLD = "compaction.memoryThreshold";
@Property(
+ longValue = GC_PROGRESS_LOG_DEFAULT,
+ label = "Compaction Progress Log",
+ description = "Enables compaction progress logging at each set of compacted nodes. A value of -1 disables the log."
+ )
+ public static final String GC_PROGRESS_LOG = "compaction.progressLog";
+
+ @Property(
boolValue = false,
label = "Standby Mode",
description = "Flag indicating that this component will not register as a NodeStore but just as a NodeStoreProvider"
@@ -657,12 +665,14 @@ public class SegmentNodeStoreService ext
log.warn("Deprecated property compaction.gainThreshold was detected. In order to configure compaction please use the new property "
+ "compaction.sizeDeltaEstimation. For turning off estimation, the new property compaction.disableEstimation should be used.");
}
+ long gcProgressLog = toLong(property(GC_PROGRESS_LOG), GC_PROGRESS_LOG_DEFAULT);
return new SegmentGCOptions(pauseCompaction, retryCount, forceTimeout)
.setRetainedGenerations(retainedGenerations)
.setGcSizeDeltaEstimation(sizeDeltaEstimation)
.setMemoryThreshold(memoryThreshold)
- .setEstimationDisabled(disableEstimation);
+ .setEstimationDisabled(disableEstimation)
+ .withGCNodeWriteMonitor(gcProgressLog);
}
private void unregisterNodeStore() {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentWriter.java Mon Dec 5 12:38:04 2016
@@ -68,6 +68,7 @@ import org.apache.jackrabbit.oak.api.Typ
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.plugins.memory.ModifiedNodeState;
import org.apache.jackrabbit.oak.segment.WriteOperationHandler.WriteOperation;
+import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
import org.apache.jackrabbit.oak.spi.state.DefaultNodeStateDiff;
@@ -101,6 +102,9 @@ public class SegmentWriter {
@Nonnull
private final WriteOperationHandler writeOperationHandler;
+ @Nonnull
+ private GCNodeWriteMonitor compactionMonitor = GCNodeWriteMonitor.EMPTY;
+
/**
* Create a new instance of a {@code SegmentWriter}. Note the thread safety properties
* pointed out in the class comment.
@@ -982,6 +986,7 @@ public class SegmentWriter {
SegmentNodeState sns = (SegmentNodeState) state;
nodeCache.put(sns.getStableId(), recordId, cost(sns));
nodeWriteStats.isCompactOp = true;
+ compactionMonitor.compacted();
}
return recordId;
}
@@ -1234,4 +1239,8 @@ public class SegmentWriter {
}
}
+ public void setCompactionMonitor(GCNodeWriteMonitor compactionMonitor) {
+ this.compactionMonitor = compactionMonitor;
+ }
+
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentGCOptions.java Mon Dec 5 12:38:04 2016
@@ -21,6 +21,8 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.base.Preconditions.checkArgument;
+import org.apache.jackrabbit.oak.segment.file.GCNodeWriteMonitor;
+
/**
* This class holds configuration options for segment store revision gc.
*/
@@ -57,6 +59,11 @@ public class SegmentGCOptions {
public static final long SIZE_DELTA_ESTIMATION_DEFAULT = 10737418240L;
/**
+ * Default value for the gc progress log
+ */
+ public static final long GC_PROGRESS_LOG_DEFAULT = -1;
+
+ /**
* Default value for {@link #getMemoryThreshold()}
*/
public static final int MEMORY_THRESHOLD_DEFAULT = 15;
@@ -89,6 +96,12 @@ public class SegmentGCOptions {
"oak.segment.compaction.gcSizeDeltaEstimation",
SIZE_DELTA_ESTIMATION_DEFAULT);
+ /**
+ * Responsible for monitoring progress of the online compaction, and
+ * providing progress tracking.
+ */
+ private GCNodeWriteMonitor gcNodeWriteMonitor = GCNodeWriteMonitor.EMPTY;
+
public SegmentGCOptions(boolean paused, int retryCount, int forceTimeout) {
this.paused = paused;
this.retryCount = retryCount;
@@ -317,4 +330,20 @@ public class SegmentGCOptions {
this.estimationDisabled = disabled;
return this;
}
+
+ /**
+ * Enables the GcWriteMonitor with the given params.
+ * @param gcProgressLog
+ * Enables compaction progress logging at each set of compacted nodes, disabled if set to
+ * {@code -1}
+ * @return this instance
+ */
+ public SegmentGCOptions withGCNodeWriteMonitor(long gcProgressLog) {
+ this.gcNodeWriteMonitor = new GCNodeWriteMonitor(gcProgressLog);
+ return this;
+ }
+
+ public GCNodeWriteMonitor getGCNodeWriteMonitor() {
+ return gcNodeWriteMonitor;
+ }
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGC.java Mon Dec 5 12:38:04 2016
@@ -170,4 +170,38 @@ public interface SegmentRevisionGC {
* @param memoryThreshold
*/
void setMemoryThreshold(int memoryThreshold);
+
+ /**
+ * @return {@code true} if there is an online compaction cycle running
+ */
+ boolean isRevisionGCRunning();
+
+ /**
+ * @return number of compacted nodes in the current cycle
+ */
+ long getCompactedNodes();
+
+ /**
+ * @return number of estimated nodes to be compacted in the current cycle.
+ * Can be {@code -1} if the estimation can't be performed
+ */
+ long getEstimatedCompactableNodes();
+
+ /**
+ * @return percentage of progress for the current compaction cycle. Can be
+ * {@code -1} if the estimation can't be performed.
+ */
+ int getEstimatedRevisionGCCompletion();
+
+ /**
+ * @return Number of nodes the monitor will log a message, {@code -1} means disabled
+ */
+ public long getRevisionGCProgressLog();
+
+ /**
+ * Set the size of the logging interval, {@code -1} means disabled
+ * @param logCycle
+ * number of nodes
+ */
+ public void setRevisionGCProgressLog(long gcProgressLog);
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/compaction/SegmentRevisionGCMBean.java Mon Dec 5 12:38:04 2016
@@ -171,4 +171,34 @@ public class SegmentRevisionGCMBean
public void setMemoryThreshold(int memoryThreshold) {
gcOptions.setMemoryThreshold(memoryThreshold);
}
+
+ @Override
+ public boolean isRevisionGCRunning() {
+ return gcOptions.getGCNodeWriteMonitor().isCompactionRunning();
+ }
+
+ @Override
+ public long getCompactedNodes() {
+ return gcOptions.getGCNodeWriteMonitor().getCompactedNodes();
+ }
+
+ @Override
+ public long getEstimatedCompactableNodes() {
+ return gcOptions.getGCNodeWriteMonitor().getEstimatedTotal();
+ }
+
+ @Override
+ public int getEstimatedRevisionGCCompletion() {
+ return gcOptions.getGCNodeWriteMonitor().getEstimatedPercentage();
+ }
+
+ @Override
+ public long getRevisionGCProgressLog() {
+ return gcOptions.getGCNodeWriteMonitor().getGcProgressLog();
+ }
+
+ @Override
+ public void setRevisionGCProgressLog(long gcProgressLog) {
+ gcOptions.getGCNodeWriteMonitor().setGcProgressLog(gcProgressLog);
+ }
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Mon Dec 5 12:38:04 2016
@@ -86,6 +86,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.SegmentWriter;
import org.apache.jackrabbit.oak.segment.WriterCacheManager;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
+import org.apache.jackrabbit.oak.segment.file.GCJournal.GCJournalEntry;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.slf4j.Logger;
@@ -713,6 +714,9 @@ public class FileStore extends AbstractF
private volatile boolean cancelled;
+ @Nonnull
+ private final GCNodeWriteMonitor compactionMonitor;
+
GarbageCollector(
@Nonnull SegmentGCOptions gcOptions,
@Nonnull GCListener gcListener,
@@ -722,6 +726,7 @@ public class FileStore extends AbstractF
this.gcListener = gcListener;
this.gcJournal = gcJournal;
this.cacheManager = cacheManager;
+ this.compactionMonitor = gcOptions.getGCNodeWriteMonitor();
}
synchronized void run() throws IOException {
@@ -778,6 +783,7 @@ public class FileStore extends AbstractF
}
gcMemoryBarrier.close();
} finally {
+ compactionMonitor.finished();
gcListener.updateStatus(IDLE.message());
}
}
@@ -806,6 +812,10 @@ public class FileStore extends AbstractF
gcListener.info("TarMK GC #{}: compaction started, gc options={}", GC_COUNT, gcOptions);
gcListener.updateStatus(COMPACTION.message());
+ GCJournalEntry gcEntry = gcJournal.read();
+ long initialSize = size();
+ compactionMonitor.init(GC_COUNT.get(), gcEntry.getRepoSize(), gcEntry.getNodes(), initialSize);
+
SegmentNodeState before = getHead();
Supplier<Boolean> cancel = new CancelCompactionSupplier(FileStore.this);
SegmentWriter writer = segmentWriterBuilder("c")
@@ -813,6 +823,7 @@ public class FileStore extends AbstractF
.withGeneration(newGeneration)
.withoutWriterPool()
.build(FileStore.this);
+ writer.setCompactionMonitor(compactionMonitor);
SegmentNodeState after = compact(before, writer, cancel);
if (after == null) {
@@ -1103,7 +1114,7 @@ public class FileStore extends AbstractF
long finalSize = size();
long reclaimedSize = initialSize - afterCleanupSize;
stats.reclaimed(reclaimedSize);
- gcJournal.persist(reclaimedSize, finalSize, getGcGeneration());
+ gcJournal.persist(reclaimedSize, finalSize, getGcGeneration(), compactionMonitor.getCompactedNodes());
gcListener.cleaned(reclaimedSize, finalSize);
gcListener.info("TarMK GC #{}: cleanup completed in {} ({} ms). Post cleanup size is {} ({} bytes)" +
" and space reclaimed {} ({} bytes).",
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java Mon Dec 5 12:38:04 2016
@@ -43,7 +43,7 @@ import org.slf4j.LoggerFactory;
/**
* Persists the repository size and the reclaimed size following a cleanup
* operation in the {@link #GC_JOURNAL gc journal} file with the format:
- * 'repoSize, reclaimedSize, timestamp, gcGen'.
+ * 'repoSize, reclaimedSize, timestamp, gcGen, nodes compacted'.
*/
public class GCJournal {
@@ -61,11 +61,18 @@ public class GCJournal {
}
/**
- * Persists the repository size and the reclaimed size following a cleanup
- * operation
+ * Persists the repository stats (current size, reclaimed size, gc
+ * generation, number of compacted nodes) following a cleanup operation for
+ * a successful compaction. NOOP if the gcGeneration is the same as the one
+ * persisted previously.
+ *
+ * @param reclaimedSize size reclaimed by cleanup
+ * @param repoSize current repo size
+ * @param gcGeneration gc generation
+ * @param nodes number of compacted nodes
*/
public synchronized void persist(long reclaimedSize, long repoSize,
- int gcGeneration) {
+ int gcGeneration, long nodes) {
GCJournalEntry current = read();
if (current.getGcGeneration() == gcGeneration) {
// failed compaction, only update the journal if the generation
@@ -73,7 +80,7 @@ public class GCJournal {
return;
}
latest = new GCJournalEntry(repoSize, reclaimedSize,
- System.currentTimeMillis(), gcGeneration);
+ System.currentTimeMillis(), gcGeneration, nodes);
Path path = new File(directory, GC_JOURNAL).toPath();
try {
try (BufferedWriter w = newBufferedWriter(path, UTF_8, WRITE,
@@ -127,43 +134,43 @@ public class GCJournal {
static class GCJournalEntry {
- static GCJournalEntry EMPTY = new GCJournalEntry(-1, -1, -1, -1);
+ static GCJournalEntry EMPTY = new GCJournalEntry(-1, -1, -1, -1, -1);
private final long repoSize;
private final long reclaimedSize;
private final long ts;
private final int gcGeneration;
+ private final long nodes;
public GCJournalEntry(long repoSize, long reclaimedSize, long ts,
- int gcGeneration) {
+ int gcGeneration, long nodes) {
this.repoSize = repoSize;
this.reclaimedSize = reclaimedSize;
this.ts = ts;
this.gcGeneration = gcGeneration;
+ this.nodes = nodes;
}
@Override
public String toString() {
- return repoSize + "," + reclaimedSize + "," + ts + ","
- + gcGeneration;
+ return repoSize + "," + reclaimedSize + "," + ts + "," + gcGeneration + "," + nodes;
}
static GCJournalEntry fromString(String in) {
String[] items = in.split(",");
- if (items.length == 3 || items.length == 4) {
- long repoSize = safeParse(items[0]);
- long reclaimedSize = safeParse(items[1]);
- long ts = safeParse(items[2]);
- int gcGen = -1;
- if (items.length == 4) {
- gcGen = (int) safeParse(items[3]);
- }
- return new GCJournalEntry(repoSize, reclaimedSize, ts, gcGen);
- }
- return GCJournalEntry.EMPTY;
+ long repoSize = safeParse(items, 0);
+ long reclaimedSize = safeParse(items, 1);
+ long ts = safeParse(items, 2);
+ int gcGen = (int) safeParse(items, 3);
+ long nodes = safeParse(items, 4);
+ return new GCJournalEntry(repoSize, reclaimedSize, ts, gcGen, nodes);
}
- private static long safeParse(String in) {
+ private static long safeParse(String[] items, int index) {
+ if (items.length < index - 1) {
+ return -1;
+ }
+ String in = items[index];
try {
return Long.parseLong(in);
} catch (NumberFormatException ex) {
@@ -172,29 +179,48 @@ public class GCJournal {
return -1;
}
+ /**
+ * Returns the repository size
+ */
public long getRepoSize() {
return repoSize;
}
+ /**
+ * Returns the reclaimed size
+ */
public long getReclaimedSize() {
return reclaimedSize;
}
+ /**
+ * Returns the timestamp
+ */
public long getTs() {
return ts;
}
+ /**
+ * Returns the gc generation
+ */
public int getGcGeneration() {
return gcGeneration;
}
+ /**
+ * Returns the number of compacted nodes
+ */
+ public long getNodes() {
+ return nodes;
+ }
+
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + gcGeneration;
- result = prime * result
- + (int) (reclaimedSize ^ (reclaimedSize >>> 32));
+ result = prime * result + (int) (nodes ^ (nodes >>> 32));
+ result = prime * result + (int) (reclaimedSize ^ (reclaimedSize >>> 32));
result = prime * result + (int) (repoSize ^ (repoSize >>> 32));
result = prime * result + (int) (ts ^ (ts >>> 32));
return result;
@@ -211,6 +237,8 @@ public class GCJournal {
GCJournalEntry other = (GCJournalEntry) obj;
if (gcGeneration != other.gcGeneration)
return false;
+ if (nodes != other.nodes)
+ return false;
if (reclaimedSize != other.reclaimedSize)
return false;
if (repoSize != other.repoSize)
@@ -219,6 +247,5 @@ public class GCJournal {
return false;
return true;
}
-
}
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java?rev=1772656&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java Mon Dec 5 12:38:04 2016
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Monitors the compaction cycle and keeps a compacted nodes counter, in order
+ * to provide a best effort progress log based on extrapolating the previous
+ * size and node count and current size to deduce current node count.
+ */
+public class GCNodeWriteMonitor {
+
+ private static final Logger log = LoggerFactory.getLogger(GCNodeWriteMonitor.class);
+
+ public static final GCNodeWriteMonitor EMPTY = new GCNodeWriteMonitor(-1);
+
+ /**
+ * Number of nodes the monitor will log a message, -1 to disable
+ */
+ private long gcProgressLog;
+
+ /**
+ * Start timestamp of compaction (reset at each {@code init()} call).
+ */
+ private long start = 0;
+
+ /**
+ * Estimated nodes to compact per cycle (reset at each {@code init()} call).
+ */
+ private long estimated = -1;
+
+ /**
+ * Compacted nodes per cycle (reset at each {@code init()} call).
+ */
+ private long nodes;
+
+ private boolean running = false;
+
+ private long gcCount;
+
+ public GCNodeWriteMonitor(long gcProgressLog) {
+ this.gcProgressLog = gcProgressLog;
+ }
+
+ public synchronized void compacted() {
+ nodes++;
+ if (gcProgressLog > 0 && nodes % gcProgressLog == 0) {
+ long ms = System.currentTimeMillis() - start;
+ log.info("TarMK GC #{}: compacted {} nodes in {} ms.", gcCount, nodes, ms);
+ start = System.currentTimeMillis();
+ }
+ }
+
+ /**
+ * @param gcCount
+ * current gc run
+ * @param prevSize
+ * size from latest successful compaction
+ * @param prevCompactedNodes
+ * number of nodes compacted during latest compaction operation
+ * @param currentSize
+ * current repository size
+ */
+ public synchronized void init(long gcCount, long prevSize, long prevCompactedNodes, long currentSize) {
+ this.gcCount = gcCount;
+ if (prevCompactedNodes > 0) {
+ estimated = (long) (((double) currentSize / prevSize) * prevCompactedNodes);
+ log.info(
+ "TarMK GC #{}: estimated number of nodes to compact is {}, based on {} nodes compacted to {} bytes "
+ + "on disk in previous compaction and current size of {} bytes on disk.",
+ this.gcCount, estimated, prevCompactedNodes, prevSize, currentSize);
+ } else {
+ log.info("TarMK GC #{}: unable to estimate number of nodes for compaction, missing gc history.");
+ }
+ nodes = 0;
+ start = System.currentTimeMillis();
+ running = true;
+ }
+
+ public synchronized void finished() {
+ running = false;
+ }
+
+ /**
+ * Compacted nodes in current cycle
+ */
+ public synchronized long getCompactedNodes() {
+ return nodes;
+ }
+
+ /**
+ * Estimated nodes to compact in current cycle. Can be {@code -1} if the
+ * estimation could not be performed.
+ */
+ public synchronized long getEstimatedTotal() {
+ return estimated;
+ }
+
+ /**
+ * Estimated completion percentage. Can be {@code -1} if the estimation
+ * could not be performed.
+ */
+ public synchronized int getEstimatedPercentage() {
+ if (estimated > 0) {
+ if (!running) {
+ return 100;
+ } else {
+ return Math.min((int) (100 * ((double) nodes / estimated)), 99);
+ }
+ }
+ return -1;
+ }
+
+ public synchronized boolean isCompactionRunning() {
+ return running;
+ }
+
+ public long getGcProgressLog() {
+ return gcProgressLog;
+ }
+
+ public void setGcProgressLog(long gcProgressLog) {
+ this.gcProgressLog = gcProgressLog;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCNodeWriteMonitor.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java?rev=1772656&r1=1772655&r2=1772656&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java Mon Dec 5 12:38:04 2016
@@ -44,26 +44,30 @@ public class GcJournalTest {
File directory = segmentFolder.newFolder();
GCJournal gc = new GCJournal(directory);
- gc.persist(0, 100, 1);
+ gc.persist(0, 100, 1, 50);
GCJournalEntry e0 = gc.read();
assertEquals(100, e0.getRepoSize());
assertEquals(0, e0.getReclaimedSize());
+ assertEquals(50, e0.getNodes());
- gc.persist(0, 250, 2);
+ gc.persist(0, 250, 2, 75);
GCJournalEntry e1 = gc.read();
assertEquals(250, e1.getRepoSize());
assertEquals(0, e1.getReclaimedSize());
+ assertEquals(75, e1.getNodes());
- gc.persist(50, 200, 3);
+ gc.persist(50, 200, 3, 90);
GCJournalEntry e2 = gc.read();
assertEquals(200, e2.getRepoSize());
assertEquals(50, e2.getReclaimedSize());
+ assertEquals(90, e2.getNodes());
// same gen
- gc.persist(75, 300, 3);
+ gc.persist(75, 300, 3, 125);
GCJournalEntry e3 = gc.read();
assertEquals(200, e3.getRepoSize());
assertEquals(50, e3.getReclaimedSize());
+ assertEquals(90, e3.getNodes());
Collection<GCJournalEntry> all = gc.readAll();
assertEquals(all.size(), 3);