You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by al...@apache.org on 2014/08/25 11:18:54 UTC
svn commit: r1620284 - in /jackrabbit/oak/branches/1.0: ./
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/
Author: alexparvulescu
Date: Mon Aug 25 09:18:53 2014
New Revision: 1620284
URL: http://svn.apache.org/r1620284
Log:
OAK-2019 Compact only if needed
- merged rev 1619411, 1619800, 1619808
Added:
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java
- copied, changed from r1619411, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntryVisitor.java
- copied unchanged from r1619411, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarEntryVisitor.java
Modified:
jackrabbit/oak/branches/1.0/ (props changed)
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
Propchange: jackrabbit/oak/branches/1.0/
------------------------------------------------------------------------------
Merged /jackrabbit/oak/trunk:r1619411,1619800,1619808
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java?rev=1620284&r1=1620283&r2=1620284&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentBlob.java Mon Aug 25 09:18:53 2014
@@ -17,6 +17,8 @@
package org.apache.jackrabbit.oak.plugins.segment;
import static com.google.common.base.Charsets.UTF_8;
+import static com.google.common.collect.Sets.newIdentityHashSet;
+import static java.util.Collections.emptySet;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.MEDIUM_LIMIT;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.SMALL_LIMIT;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentWriter.BLOCK_SIZE;
@@ -24,6 +26,7 @@ import static org.apache.jackrabbit.oak.
import java.io.BufferedInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.util.Set;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
@@ -34,6 +37,14 @@ import org.apache.jackrabbit.oak.spi.blo
public class SegmentBlob extends Record implements Blob {
+ public static Iterable<SegmentId> getBulkSegmentIds(Blob blob) {
+ if (blob instanceof SegmentBlob) {
+ return ((SegmentBlob) blob).getBulkSegmentIds();
+ } else {
+ return emptySet();
+ }
+ }
+
SegmentBlob(RecordId id) {
super(id);
}
@@ -204,4 +215,24 @@ public class SegmentBlob extends Record
return new String(bytes, UTF_8);
}
+ private Iterable<SegmentId> getBulkSegmentIds() {
+ Segment segment = getSegment();
+ int offset = getOffset();
+ byte head = segment.readByte(offset);
+ if ((head & 0xe0) == 0xc0) {
+ // 110x xxxx: long value
+ long length = (segment.readLong(offset) & 0x1fffffffffffffffL) + MEDIUM_LIMIT;
+ int listSize = (int) ((length + BLOCK_SIZE - 1) / BLOCK_SIZE);
+ ListRecord list = new ListRecord(
+ segment.readRecordId(offset + 8), listSize);
+ Set<SegmentId> ids = newIdentityHashSet();
+ for (RecordId id : list.getEntries()) {
+ ids.add(id.getSegmentId());
+ }
+ return ids;
+ } else {
+ return emptySet();
+ }
+ }
+
}
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java?rev=1620284&r1=1620283&r2=1620284&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentNodeStoreService.java Mon Aug 25 09:18:53 2014
@@ -17,6 +17,7 @@
package org.apache.jackrabbit.oak.plugins.segment;
import static com.google.common.base.Preconditions.checkState;
+import static org.apache.jackrabbit.oak.commons.PropertiesUtil.toBoolean;
import static org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.registerMBean;
import java.io.Closeable;
@@ -76,6 +77,9 @@ public class SegmentNodeStoreService ext
@Property(description="Cache size (MB)", intValue=256)
public static final String CACHE = "cache";
+ @Property(description = "TarMK compaction paused flag", boolValue = true)
+ public static final String PAUSE_COMPACTION = "pauseCompaction";
+
/**
* Boolean value indicating a blobStore is to be used
*/
@@ -148,10 +152,12 @@ public class SegmentNodeStoreService ext
size = System.getProperty(SIZE, "256");
}
+ boolean pauseCompaction = toBoolean(lookup(context, PAUSE_COMPACTION), true);
store = new FileStore(
blobStore,
new File(directory),
- Integer.parseInt(size), "64".equals(mode));
+ Integer.parseInt(size), "64".equals(mode))
+ .setPauseCompaction(pauseCompaction);
delegate = new SegmentNodeStore(store);
observerTracker = new ObserverTracker(delegate);
Copied: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java (from r1619411, jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java)
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java?p2=jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java&p1=jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java&r1=1619411&r2=1620284&rev=1620284&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionGainEstimate.java Mon Aug 25 09:18:53 2014
@@ -75,6 +75,9 @@ class CompactionGainEstimate implements
* @return percentage of disk space that could be freed with compaction
*/
public long estimateCompactionGain() {
+ if (totalSize == 0) {
+ return 0;
+ }
return 100 * (totalSize - reachableSize) / totalSize;
}
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1620284&r1=1620283&r2=1620284&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Mon Aug 25 09:18:53 2014
@@ -131,7 +131,14 @@ public class FileStore implements Segmen
private final AtomicBoolean cleanupNeeded = new AtomicBoolean(false);
/**
- * List of old tar file generations that are waiting to be removed.
+ * Flag to set the compaction on pause.
+ */
+ private volatile boolean pauseCompaction = true;
+
+ /**
+ * List of old tar file generations that are waiting to be removed. They can
+ * not be removed immediately, because they first need to be closed, and the
+ * JVM needs to release the memory mapped file references.
*/
private final LinkedList<File> toBeRemoved = newLinkedList();
@@ -244,7 +251,30 @@ public class FileStore implements Segmen
new Runnable() {
@Override
public void run() {
- compact();
+ log.info("TarMK compaction started");
+ long time = System.currentTimeMillis();
+ CompactionGainEstimate estimate = estimateCompactionGain();
+ long gain = estimate.estimateCompactionGain();
+ time = System.currentTimeMillis() - time;
+ if (gain >= 10) {
+ log.info(
+ "Estimated compaction in {}ms, gain is {}% ({}/{}), so running compaction",
+ new Object[] { time, gain,
+ estimate.getReachableSize(),
+ estimate.getTotalSize() });
+ if (!pauseCompaction) {
+ compact();
+ } else {
+ log.info("TarMK compaction paused");
+ }
+ } else {
+ log.info(
+ "Estimated compaction in {}ms, gain is {}% ({}/{}), so skipping compaction for now",
+ new Object[] { time, gain,
+ estimate.getReachableSize(),
+ estimate.getTotalSize() });
+ }
+ cleanupNeeded.set(true);
}
});
@@ -331,6 +361,30 @@ public class FileStore implements Segmen
return size;
}
+ /**
+ * Returns the number of segments in this TarMK instance.
+ *
+ * @return number of segments
+ */
+ private synchronized int count() {
+ int count = writer.count();
+ for (TarReader reader : readers) {
+ count += reader.count();
+ }
+ return count;
+ }
+
+ CompactionGainEstimate estimateCompactionGain() {
+ CompactionGainEstimate estimate = new CompactionGainEstimate(getHead(),
+ count());
+ synchronized (this) {
+ for (TarReader reader : readers) {
+ reader.accept(estimate);
+ }
+ }
+ return estimate;
+ }
+
public void flush() throws IOException {
synchronized (persistedHead) {
RecordId before = persistedHead.get();
@@ -411,7 +465,7 @@ public class FileStore implements Segmen
public void compact() {
long start = System.nanoTime();
- log.info("TarMK compaction started");
+ log.info("TarMK compaction running");
SegmentWriter writer = new SegmentWriter(this, tracker);
Compactor compactor = new Compactor(writer);
@@ -446,7 +500,6 @@ public class FileStore implements Segmen
log.info("TarMK compaction completed in {}ms", MILLISECONDS
.convert(System.nanoTime() - start, NANOSECONDS));
- cleanupNeeded.set(true);
}
public synchronized Iterable<SegmentId> getSegmentIds() {
@@ -640,4 +693,8 @@ public class FileStore implements Segmen
compactionThread.trigger();
}
+ public FileStore setPauseCompaction(boolean pauseCompaction) {
+ this.pauseCompaction = pauseCompaction;
+ return this;
+ }
}
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1620284&r1=1620283&r2=1620284&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Mon Aug 25 09:18:53 2014
@@ -67,7 +67,7 @@ class TarReader {
/** The tar file block size. */
private static final int BLOCK_SIZE = TarWriter.BLOCK_SIZE;
- private static final int getEntrySize(int size) {
+ static int getEntrySize(int size) {
return BLOCK_SIZE + size + TarWriter.getPaddingSize(size);
}
@@ -485,6 +485,34 @@ class TarReader {
return file.length();
}
+ /**
+ * Returns the number of segments in this tar file.
+ *
+ * @return number of segments
+ */
+ int count() {
+ return index.capacity() / 24;
+ }
+
+ /**
+ * Iterates over all entries in this tar file and calls
+ * {@link TarEntryVisitor#visit(long, long, File, int, int)} on them.
+ *
+ * @param visitor entry visitor
+ */
+ void accept(TarEntryVisitor visitor) {
+ int position = index.position();
+ while (position < index.limit()) {
+ visitor.visit(
+ index.getLong(position),
+ index.getLong(position + 8),
+ file,
+ index.getInt(position + 16),
+ index.getInt(position + 20));
+ position += 24;
+ }
+ }
+
Set<UUID> getUUIDs() {
Set<UUID> uuids = newHashSetWithExpectedSize(index.remaining() / 24);
int position = index.position();
Modified: jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java?rev=1620284&r1=1620283&r2=1620284&view=diff
==============================================================================
--- jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java (original)
+++ jackrabbit/oak/branches/1.0/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarWriter.java Mon Aug 25 09:18:53 2014
@@ -111,6 +111,15 @@ class TarWriter {
this.file = file;
}
+ /**
+ * Returns the number of segments written so far to this tar file.
+ *
+ * @return number of segments written so far
+ */
+ synchronized int count() {
+ return index.size();
+ }
+
synchronized Set<UUID> getUUIDs() {
return newHashSet(index.keySet());
}