You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2015/05/18 10:32:29 UTC
svn commit: r1679958 -
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
Author: mduerig
Date: Mon May 18 08:32:28 2015
New Revision: 1679958
URL: http://svn.apache.org/r1679958
Log:
OAK-2713: High memory usage of CompactionMap
compress with non empty remove set forgets mappings
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java?rev=1679958&r1=1679957&r2=1679958&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMap.java Mon May 18 08:32:28 2015
@@ -82,8 +82,8 @@ public class CompactionMap {
private short[] afterOffsets = new short[0];
private int[] afterSegmentIds = new int[0];
- private long[] amsbs = new long[0];
- private long[] alsbs = new long[0];
+ private long[] afterMsbs = new long[0];
+ private long[] afterLsbs = new long[0];
private long prevWeight;
private CompactionMap prev;
@@ -115,13 +115,8 @@ public class CompactionMap {
private boolean recursiveWasCompactedTo(RecordId before,
RecordId after) {
RecordId potentialAfter = recursiveGet(this, before);
- if (potentialAfter == null) {
- return false;
- }
- if (after.equals(potentialAfter)) {
- return true;
- }
- return recursiveWasCompactedTo(potentialAfter, after);
+ return potentialAfter != null &&
+ (after.equals(potentialAfter) || recursiveWasCompactedTo(potentialAfter, after));
}
private static RecordId recursiveGet(CompactionMap map, RecordId before) {
@@ -149,14 +144,8 @@ public class CompactionMap {
}
private static boolean wasCompacted(CompactionMap map, long msb, long lsb) {
- int find = map.findEntry(msb, lsb);
- if (find != -1) {
- return true;
- }
- if (map.prev != null) {
- return wasCompacted(map.prev, msb, lsb);
- }
- return false;
+ return map.findEntry(msb, lsb) != -1 ||
+ map.prev != null && wasCompacted(map.prev, msb, lsb);
}
public RecordId get(RecordId before) {
@@ -203,7 +192,7 @@ public class CompactionMap {
private SegmentId asSegmentId(int index) {
int idx = afterSegmentIds[index];
- return new SegmentId(tracker, amsbs[idx], alsbs[idx]);
+ return new SegmentId(tracker, afterMsbs[idx], afterLsbs[idx]);
}
private static UUID asUUID(SegmentId id) {
@@ -247,11 +236,12 @@ public class CompactionMap {
private void compressInternal(Set<UUID> removed) {
if (recent.isEmpty() && removed.isEmpty()) {
- // noop
+ // no-op
return;
}
- Set<UUID> uuids = newTreeSet();
+ Set<UUID> uuids = newTreeSet();
+ int newSize = 0;
Map<UUID, Map<Integer, RecordId>> mapping = newTreeMap();
for (Entry<RecordId, RecordId> entry : recent.entrySet()) {
RecordId before = entry.getKey();
@@ -260,7 +250,9 @@ public class CompactionMap {
UUID uuid = new UUID(
id.getMostSignificantBits(),
id.getLeastSignificantBits());
- uuids.add(uuid);
+ if (uuids.add(uuid) && !removed.contains(uuid)) {
+ newSize++;
+ }
Map<Integer, RecordId> map = mapping.get(uuid);
if (map == null) {
@@ -271,15 +263,15 @@ public class CompactionMap {
}
for (int i = 0; i < msbs.length; i++) {
- uuids.add(new UUID(msbs[i], lsbs[i]));
+ UUID uuid = new UUID(msbs[i], lsbs[i]);
+ if (uuids.add(uuid) && !removed.contains(uuid)) {
+ newSize++;
+ }
}
- uuids.removeAll(removed);
- mapping.keySet().removeAll(removed);
-
- long[] newmsbs = new long[uuids.size()];
- long[] newlsbs = new long[uuids.size()];
- int[] newEntryIndex = new int[uuids.size() + 1];
+ long[] newMsbs = new long[newSize];
+ long[] newLsbs = new long[newSize];
+ int[] newEntryIndex = new int[newSize + 1];
int newEntries = beforeOffsets.length + recent.size();
short[] newBeforeOffsets = new short[newEntries];
@@ -292,36 +284,47 @@ public class CompactionMap {
int newEntry = 0;
int oldEntry = 0;
for (UUID uuid : uuids) {
- newmsbs[newEntry] = uuid.getMostSignificantBits();
- newlsbs[newEntry] = uuid.getLeastSignificantBits();
+ long msb = uuid.getMostSignificantBits();
+ long lsb = uuid.getLeastSignificantBits();
+
+ if (removed.contains(uuid)) {
+ if (oldEntry < msbs.length
+ && msbs[oldEntry] == msb
+ && lsbs[oldEntry] == lsb) {
+ oldEntry++;
+ }
+ continue;
+ }
// offset -> record
- Map<Integer, RecordId> newsegment = mapping.get(uuid);
- if (newsegment == null) {
- newsegment = newTreeMap();
+ Map<Integer, RecordId> newSegment = mapping.get(uuid);
+ if (newSegment == null) {
+ newSegment = newTreeMap();
}
if (oldEntry < msbs.length
- && msbs[oldEntry] == newmsbs[newEntry]
- && lsbs[oldEntry] == newlsbs[newEntry]) {
+ && msbs[oldEntry] == msb
+ && lsbs[oldEntry] == lsb) {
int index = entryIndex[oldEntry];
int limit = entryIndex[oldEntry + 1];
for (int i = index; i < limit; i++) {
- newsegment.put(decode(beforeOffsets[i]), new RecordId(
+ newSegment.put(decode(beforeOffsets[i]), new RecordId(
asSegmentId(i), decode(afterOffsets[i])));
}
oldEntry++;
}
+ newMsbs[newEntry] = msb;
+ newLsbs[newEntry] = lsb;
newEntryIndex[newEntry++] = newIndex;
- for (Entry<Integer, RecordId> entry : newsegment.entrySet()) {
+ for (Entry<Integer, RecordId> entry : newSegment.entrySet()) {
int key = entry.getKey();
RecordId id = entry.getValue();
newBeforeOffsets[newIndex] = encode(key);
newAfterOffsets[newIndex] = encode(id.getOffset());
UUID aUUID = asUUID(id.getSegmentId());
- int aSIdx = -1;
+ int aSIdx;
if (newAfterSegments.containsKey(aUUID)) {
aSIdx = newAfterSegments.get(aUUID);
} else {
@@ -336,8 +339,8 @@ public class CompactionMap {
newEntryIndex[newEntry] = newIndex;
- this.msbs = newmsbs;
- this.lsbs = newlsbs;
+ this.msbs = newMsbs;
+ this.lsbs = newLsbs;
this.entryIndex = newEntryIndex;
if (newIndex < newBeforeOffsets.length) {
@@ -350,12 +353,12 @@ public class CompactionMap {
this.afterSegmentIds = newAfterSegmentIds;
}
- this.amsbs = new long[newAfterSegments.size()];
- this.alsbs = new long[newAfterSegments.size()];
+ this.afterMsbs = new long[newAfterSegments.size()];
+ this.afterLsbs = new long[newAfterSegments.size()];
for (Entry<UUID, Integer> entry : newAfterSegments.entrySet()) {
- this.amsbs[entry.getValue()] = entry.getKey()
+ this.afterMsbs[entry.getValue()] = entry.getKey()
.getMostSignificantBits();
- this.alsbs[entry.getValue()] = entry.getKey()
+ this.afterLsbs[entry.getValue()] = entry.getKey()
.getLeastSignificantBits();
}
@@ -437,7 +440,7 @@ public class CompactionMap {
StringBuilder sb = new StringBuilder();
CompactionMap cm = this;
while (cm != null) {
- sb.append("[");
+ sb.append('[');
sb.append(getCompactionStats(cm));
sb.append("], ");
cm = cm.prev;
@@ -446,14 +449,12 @@ public class CompactionMap {
}
private static String getCompactionStats(CompactionMap cm) {
- StringBuilder sb = new StringBuilder();
- sb.append("Estimated Weight: ");
- sb.append(humanReadableByteCount(getEstimatedWeight(cm)));
- sb.append(", Records: ");
- sb.append(cm.afterOffsets.length);
- sb.append(", Segments: ");
- sb.append(cm.amsbs.length);
- return sb.toString();
+ return "Estimated Weight: " +
+ humanReadableByteCount(getEstimatedWeight(cm)) +
+ ", Records: " +
+ cm.afterOffsets.length +
+ ", Segments: " +
+ cm.afterMsbs.length;
}
/**
@@ -506,10 +507,10 @@ public class CompactionMap {
// afterSegmentIds
total += 24 + cm.afterSegmentIds.length * 4;
- // amsbs
- total += 24 + cm.amsbs.length * 8;
- // alsbs
- total += 24 + cm.alsbs.length * 8;
+ // afterMsbs
+ total += 24 + cm.afterMsbs.length * 8;
+ // afterLsbs
+ total += 24 + cm.afterLsbs.length * 8;
return total;
}