You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2015/04/15 16:43:04 UTC
svn commit: r1673791 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment:
CompactionMap.java file/FileStore.java file/TarReader.java
Author: mduerig
Date: Wed Apr 15 14:43:03 2015
New Revision: 1673791
URL: http://svn.apache.org/r1673791
Log:
OAK-2713: High memory usage of CompactionMap
Cleanup compaction map after FileStore cleanup
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java?rev=1673791&r1=1673790&r2=1673791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java Wed Apr 15 14:43:03 2015
@@ -22,6 +22,8 @@ import static com.google.common.collect.
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ALIGN_BITS;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
@@ -223,8 +225,28 @@ public class CompactionMap {
}
}
- void compress() {
- if (recent.isEmpty()) {
+ public void compress(Set<UUID> removed) {
+ CompactionMap cm = this;
+ while (cm != null) {
+ cm.compressInternal(removed);
+ cm = cm.prev;
+ }
+
+ cm = this;
+ while (cm != null) {
+ while (cm.prev != null && cm.prev.msbs.length == 0) {
+ cm.prev = cm.prev.prev;
+ }
+ cm = cm.prev;
+ }
+ }
+
+ public void compress() {
+ compressInternal(Collections.<UUID>emptySet());
+ }
+
+ private void compressInternal(Set<UUID> removed) {
+ if (recent.isEmpty() && removed.isEmpty()) {
// noop
return;
}
@@ -252,6 +274,9 @@ public class CompactionMap {
uuids.add(new UUID(msbs[i], lsbs[i]));
}
+ uuids.removeAll(removed);
+ mapping.keySet().removeAll(removed);
+
long[] newmsbs = new long[uuids.size()];
long[] newlsbs = new long[uuids.size()];
int[] newEntryIndex = new int[uuids.size() + 1];
@@ -315,10 +340,16 @@ public class CompactionMap {
this.lsbs = newlsbs;
this.entryIndex = newEntryIndex;
- this.beforeOffsets = newBeforeOffsets;
- this.afterOffsets = newAfterOffsets;
+ if (newIndex < newBeforeOffsets.length) {
+ this.beforeOffsets = Arrays.copyOf(newBeforeOffsets, newIndex);
+ this.afterOffsets = Arrays.copyOf(newAfterOffsets, newIndex);
+ this.afterSegmentIds = Arrays.copyOf(newAfterSegmentIds, newIndex);
+ } else {
+ this.beforeOffsets = newBeforeOffsets;
+ this.afterOffsets = newAfterOffsets;
+ this.afterSegmentIds = newAfterSegmentIds;
+ }
- this.afterSegmentIds = newAfterSegmentIds;
this.amsbs = new long[newAfterSegments.size()];
this.alsbs = new long[newAfterSegments.size()];
for (Entry<UUID, Integer> entry : newAfterSegments.entrySet()) {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1673791&r1=1673790&r2=1673791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Wed Apr 15 14:43:03 2015
@@ -655,8 +655,9 @@ public class FileStore implements Segmen
CompactionMap cm = tracker.getCompactionMap();
List<TarReader> list = newArrayListWithCapacity(readers.size());
+ Set<UUID> cleanedIds = newHashSet();
for (TarReader reader : readers) {
- TarReader cleaned = reader.cleanup(ids, cm);
+ TarReader cleaned = reader.cleanup(ids, cm, cleanedIds);
if (cleaned == reader) {
list.add(reader);
} else {
@@ -668,6 +669,7 @@ public class FileStore implements Segmen
toBeRemoved.addLast(file);
}
}
+ cm.compress(cleanedIds);
readers = list;
long finalSize = size();
gcMonitor.cleaned(initialSize - finalSize, finalSize);
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java?rev=1673791&r1=1673790&r2=1673791&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/TarReader.java Wed Apr 15 14:43:03 2015
@@ -568,10 +568,14 @@ class TarReader {
* for memory mapped files).
*
* @param referencedIds the referenced segment ids (input and output).
+ * @param cm the compaction map
+ * @param removed a set which will receive the uuids of all segments that
+ * have been cleaned.
* @return this (if the file is kept as is), or the new generation file, or
* null if the file is fully garbage
*/
- synchronized TarReader cleanup(Set<UUID> referencedIds, CompactionMap cm) throws IOException {
+ synchronized TarReader cleanup(Set<UUID> referencedIds, CompactionMap cm, Set<UUID> removed)
+ throws IOException {
Set<UUID> cleaned = newHashSet();
Map<UUID, List<UUID>> graph = getGraph();
@@ -658,6 +662,7 @@ class TarReader {
if (count == 0) {
// none of the entries within this tar file are referenceable
+ removed.addAll(cleaned);
logCleanedSegments(cleaned);
return null;
} else if (size >= access.length() * 3 / 4 && graph != null) {
@@ -695,6 +700,7 @@ class TarReader {
singletonList(newFile), access.isMemoryMapped());
if (reader != null) {
logCleanedSegments(cleaned);
+ removed.addAll(cleaned);
return reader;
} else {
log.warn("Failed to open cleaned up tar file {}", file);