You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2015/06/05 17:09:45 UTC
svn commit: r1683780 [2/2] - in /jackrabbit/oak/trunk:
oak-commons/src/main/java/org/apache/jackrabbit/oak/commons/benchmark/
oak-core/ oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
oak-core/src/main/java/org/apache/jackrabbit/oak/p...
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStore.java Fri Jun 5 15:09:44 2015
@@ -29,6 +29,7 @@ import static java.util.Collections.empt
import static java.util.Collections.singletonMap;
import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.apache.jackrabbit.oak.plugins.segment.CompactionMap.sum;
import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.NO_COMPACTION;
import java.io.File;
@@ -56,8 +57,9 @@ import com.google.common.base.Stopwatch;
import com.google.common.collect.Maps;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
-import org.apache.jackrabbit.oak.plugins.segment.CompactionMap;
import org.apache.jackrabbit.oak.plugins.segment.Compactor;
+import org.apache.jackrabbit.oak.plugins.segment.CompactionMap;
+import org.apache.jackrabbit.oak.plugins.segment.PersistedCompactionMap;
import org.apache.jackrabbit.oak.plugins.segment.RecordId;
import org.apache.jackrabbit.oak.plugins.segment.Segment;
import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
@@ -432,10 +434,10 @@ public class FileStore implements Segmen
Runtime runtime = Runtime.getRuntime();
long avail = runtime.totalMemory() - runtime.freeMemory();
- long delta = 0;
- if (compactionStrategy.getCompactionMap() != null) {
- delta = compactionStrategy.getCompactionMap().getLastWeight();
- }
+ long[] weights = tracker.getCompactionMap().getEstimatedWeights();
+ long delta = weights.length > 0
+ ? weights[0]
+ : 0;
long needed = delta * compactionStrategy.getMemoryThreshold();
if (needed >= avail) {
gcMonitor.skipped(
@@ -453,8 +455,12 @@ public class FileStore implements Segmen
compactionStrategy.setCompactionStart(System.currentTimeMillis());
boolean compacted = false;
+ long offset = compactionStrategy.getPersistCompactionMap()
+ ? sum(tracker.getCompactionMap().getRecordCounts()) * PersistedCompactionMap.BYTES_PER_ENTRY
+ : 0;
+
CompactionGainEstimate estimate = estimateCompactionGain();
- long gain = estimate.estimateCompactionGain();
+ long gain = estimate.estimateCompactionGain(offset);
if (gain >= 10) {
gcMonitor.info(
"Estimated compaction in {}, gain is {}% ({}/{}) or ({}/{}), so running compaction",
@@ -671,15 +677,16 @@ public class FileStore implements Segmen
toBeRemoved.addLast(file);
}
}
- cm.compress(cleanedIds);
readers = list;
+ cm.remove(cleanedIds);
long finalSize = size();
gcMonitor.cleaned(initialSize - finalSize, finalSize);
gcMonitor.info("TarMK revision cleanup completed in {}. Post cleanup size is {} " +
"and space reclaimed {}. Compaction map weight/depth is {}/{}.", watch,
humanReadableByteCount(finalSize),
humanReadableByteCount(initialSize - finalSize),
- humanReadableByteCount(cm.getEstimatedWeight()), cm.getDepth());
+ humanReadableByteCount(sum(cm.getEstimatedWeights())),
+ cm.getDepth());
}
/**
@@ -694,7 +701,10 @@ public class FileStore implements Segmen
long start = System.currentTimeMillis();
SegmentWriter writer = new SegmentWriter(this, tracker, getVersion());
- final Compactor compactor = new Compactor(writer, compactionStrategy.cloneBinaries());
+ SegmentWriter mapWriter = compactionStrategy.getPersistCompactionMap()
+ ? new SegmentWriter(this, tracker, getVersion())
+ : null;
+ final Compactor compactor = new Compactor(writer, mapWriter, compactionStrategy.cloneBinaries());
SegmentNodeState before = getHead();
long existing = before.getChildNode(SegmentNodeStore.CHECKPOINTS)
.getChildNodeCount(Long.MAX_VALUE);
@@ -992,6 +1002,7 @@ public class FileStore implements Segmen
public FileStore setCompactionStrategy(CompactionStrategy strategy) {
this.compactionStrategy = strategy;
+ log.info("Compaction strategy set to: {}", strategy);
return this;
}
@@ -1076,16 +1087,16 @@ public class FileStore implements Segmen
// needs to be called inside the commitSemaphore as doing otherwise
// might result in mixed segments. See OAK-2192.
if (setHead(before, after)) {
- CompactionMap cm = compactor.getCompactionMap();
- tracker.setCompactionMap(cm);
- compactionStrategy.setCompactionMap(cm);
+ tracker.setCompactionMap(compactor.getCompactionMap());
// Drop the SegmentWriter caches and flush any existing state
// in an attempt to prevent new references to old pre-compacted
- // content. TODO: There should be a cleaner way to do this.
+ // content. TODO: There should be a cleaner way to do this. (implement GCMonitor!?)
tracker.getWriter().dropCache();
tracker.getWriter().flush();
- gcMonitor.compacted();
+
+ CompactionMap cm = tracker.getCompactionMap();
+ gcMonitor.compacted(cm.getSegmentCounts(), cm.getRecordCounts(), cm.getEstimatedWeights());
tracker.clearSegmentIdTables(compactionStrategy);
return true;
} else {
@@ -1125,8 +1136,8 @@ public class FileStore implements Segmen
}
@Override
- public void compacted() {
- delegatee.compacted();
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
+ delegatee.compacted(segmentCounts, recordCounts, compactionMapWeights);
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreGCMonitor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreGCMonitor.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreGCMonitor.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/FileStoreGCMonitor.java Fri Jun 5 15:09:44 2015
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.plugin
import static com.google.common.base.Preconditions.checkNotNull;
import static java.text.DateFormat.getDateTimeInstance;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
import static org.apache.jackrabbit.stats.TimeSeriesStatsUtil.asCompositeData;
import static org.slf4j.helpers.MessageFormatter.arrayFormat;
@@ -53,6 +54,9 @@ public class FileStoreGCMonitor extends
private final Clock clock;
private long lastCompaction;
+ private long[] segmentCounts = new long[0];
+ private long[] recordCounts = new long[0];
+ private long[] compactionMapWeights = new long[0];
private long lastCleanup;
private String lastError;
private String status = "NA";
@@ -97,8 +101,11 @@ public class FileStoreGCMonitor extends
}
@Override
- public void compacted() {
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
lastCompaction = clock.getTime();
+ this.segmentCounts = segmentCounts;
+ this.recordCounts = recordCounts;
+ this.compactionMapWeights = compactionMapWeights;
}
@Override
@@ -140,6 +147,23 @@ public class FileStoreGCMonitor extends
return status;
}
+ @Override
+ public String getCompactionMapStats() {
+ StringBuilder sb = new StringBuilder();
+ String sep = "";
+ for (int k = 0; k < segmentCounts.length; k++) {
+ sb.append(sep).append('[')
+ .append("Estimated Weight: ")
+ .append(humanReadableByteCount(compactionMapWeights[k])).append(", ")
+ .append("Segments: ")
+ .append(segmentCounts[k]).append(", ")
+ .append("Records: ")
+ .append(recordCounts[k]).append(']');
+ sep = ", ";
+ }
+ return sb.toString();
+ }
+
@Nonnull
@Override
public CompositeData getRepositorySize() {
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/GCMonitorMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/GCMonitorMBean.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/GCMonitorMBean.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/file/GCMonitorMBean.java Fri Jun 5 15:09:44 2015
@@ -55,6 +55,12 @@ public interface GCMonitorMBean {
String getStatus();
/**
+ * Statistics about the compaction map.
+ */
+ @Nonnull
+ String getCompactionMapStats();
+
+ /**
* @return time series of the repository size
*/
@Nonnull
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/DelegatingGCMonitor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/DelegatingGCMonitor.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/DelegatingGCMonitor.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/DelegatingGCMonitor.java Fri Jun 5 15:09:44 2015
@@ -80,9 +80,9 @@ public class DelegatingGCMonitor impleme
}
@Override
- public void compacted() {
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
for (GCMonitor gcMonitor : gcMonitors) {
- gcMonitor.compacted();
+ gcMonitor.compacted(segmentCounts, recordCounts, compactionMapWeights);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitor.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitor.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitor.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitor.java Fri Jun 5 15:09:44 2015
@@ -61,8 +61,12 @@ public interface GCMonitor {
/**
* The compaction phase of the garbage collection process terminated successfully.
+ * @param segmentCounts number of segments in the individual generations of the map
+ * @param recordCounts number of records in the individual generations of the map
+ * @param compactionMapWeights weights of the individual generations of the map
+ * @see org.apache.jackrabbit.oak.plugins.segment.PartialCompactionMap
*/
- void compacted();
+ void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights);
/**
* The cleanup phase of the garbage collection process terminated successfully.
@@ -76,7 +80,7 @@ public interface GCMonitor {
@Override public void warn(String message, Object[] arguments) { }
@Override public void error(String message, Exception e) { }
@Override public void skipped(String reason, Object[] arguments) { }
- @Override public void compacted() { }
+ @Override public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) { }
@Override public void cleaned(long reclaimedSize, long currentSize) { }
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitorTracker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitorTracker.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitorTracker.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/GCMonitorTracker.java Fri Jun 5 15:09:44 2015
@@ -60,9 +60,9 @@ public class GCMonitorTracker extends Ab
}
@Override
- public void compacted() {
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
for (GCMonitor gcMonitor : getServices()) {
- gcMonitor.compacted();
+ gcMonitor.compacted(segmentCounts, recordCounts, compactionMapWeights);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/package-info.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/package-info.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/package-info.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/spi/gc/package-info.java Fri Jun 5 15:09:44 2015
@@ -17,7 +17,7 @@
* under the License.
*/
-@Version("1.0")
+@Version("2.0.0")
@Export(optional = "provide:=true")
package org.apache.jackrabbit.oak.spi.gc;
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionAndCleanupTest.java Fri Jun 5 15:09:44 2015
@@ -98,6 +98,9 @@ public class CompactionAndCleanupTest {
return nodeStore.locked(setHead);
}
};
+ // Use in memory compaction map as gains asserted later on
+ // do not take additional space of the compaction map into consideration
+ custom.setPersistCompactionMap(false);
fileStore.setCompactionStrategy(custom);
// 1a. Create a bunch of data
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/CompactionMapTest.java Fri Jun 5 15:09:44 2015
@@ -16,26 +16,19 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.jackrabbit.oak.plugins.segment;
import static com.google.common.collect.Iterables.get;
-import static com.google.common.collect.Lists.newArrayList;
import static com.google.common.collect.Maps.newHashMap;
-import static com.google.common.collect.Maps.newLinkedHashMap;
-import static com.google.inject.internal.util.$Sets.newHashSet;
-import static java.io.File.createTempFile;
-import static junit.framework.Assert.assertTrue;
-import static org.apache.jackrabbit.oak.commons.benchmark.MicroBenchmark.run;
-import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE;
-import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ALIGN_BITS;
+import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.jackrabbit.oak.plugins.segment.CompactionMap.sum;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
-import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assume.assumeTrue;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Iterator;
+import static org.apache.jackrabbit.oak.plugins.segment.TestUtils.randomRecordIdMap;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -43,265 +36,136 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
-import com.google.common.base.Stopwatch;
-import org.apache.jackrabbit.oak.commons.benchmark.MicroBenchmark.Benchmark;
-import org.junit.After;
-import org.junit.Before;
+import com.google.common.collect.ImmutableList;
+import org.apache.jackrabbit.oak.plugins.segment.memory.MemoryStore;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+@RunWith(Parameterized.class)
public class CompactionMapTest {
- private static final boolean BENCH = Boolean.getBoolean("benchmark");
- private static final int SEED = Integer.getInteger("SEED", new Random().nextInt());
-
- private final Random rnd = new Random(SEED);
-
- private File directory;
- private SegmentStore segmentStore;
-
- private CompactionMap map;
- private Map<RecordId, RecordId> reference;
-
- @Before
- public void setup() throws IOException {
- directory = createTempFile(CompactionMapTest.class.getSimpleName(), "dir", new File("target"));
- directory.delete();
- directory.mkdir();
-
- segmentStore = newFileStore(directory).create();
- SegmentWriter writer = new SegmentWriter(segmentStore, getTracker(), V_11);
- map = new CompactionMap(100000, writer.getTracker());
- reference = newLinkedHashMap();
- }
-
- @After
- public void tearDown() {
- segmentStore.close();
- directory.delete();
- }
-
- private SegmentTracker getTracker() {
- return segmentStore.getTracker();
- }
-
- /**
- * Returns a new valid record offset, between {@code a} and {@code b},
- * exclusive.
- */
- private static int newValidOffset(Random random, int a, int b) {
- int p = (a >> RECORD_ALIGN_BITS) + 1;
- int q = (b >> RECORD_ALIGN_BITS);
- return (p + random.nextInt(q - p)) << RECORD_ALIGN_BITS;
- }
-
- private Map<RecordId, RecordId> randomMap(int maxSegments, int maxEntriesPerSegment) {
- Map<RecordId, RecordId> map = newHashMap();
- int segments = rnd.nextInt(maxSegments);
- for (int i = 0; i < segments; i++) {
- SegmentId id = getTracker().newDataSegmentId();
- int n = rnd.nextInt(maxEntriesPerSegment);
- int offset = MAX_SEGMENT_SIZE;
- for (int j = 0; j < n; j++) {
- offset = newValidOffset(rnd, (n - j) << RECORD_ALIGN_BITS, offset);
- RecordId before = new RecordId(id, offset);
- RecordId after = new RecordId(
- getTracker().newDataSegmentId(),
- newValidOffset(rnd, 0, MAX_SEGMENT_SIZE));
- map.put(before, after);
- }
+ private final SegmentStore store = new MemoryStore();
+ private final SegmentTracker tracker = new SegmentTracker(store);
+ private final Random rnd = new Random();
+
+ private final Map<RecordId, RecordId> referenceMap1;
+ private final Map<RecordId, RecordId> referenceMap2;
+ private final Map<RecordId, RecordId> referenceMap3;
+ private final Map<RecordId, RecordId> referenceMap = newHashMap();
+
+ private final PartialCompactionMap compactionMap1;
+ private final PartialCompactionMap compactionMap2;
+ private final PartialCompactionMap compactionMap3;
+ private final CompactionMap compactionMap;
+
+ @Parameterized.Parameters
+ public static List<Boolean[]> fixtures() {
+ return ImmutableList.of(new Boolean[] {true}, new Boolean[] {false});
+ }
+
+ private static PartialCompactionMap createCompactionMap(SegmentTracker tracker, SegmentWriter writer) {
+ if (writer != null) {
+ return new PersistedCompactionMap(writer);
+ } else {
+ return new InMemoryCompactionMap(tracker);
}
- return map;
}
- private void addRandomEntries(int maxSegments, int maxEntriesPerSegment) {
- for (Entry<RecordId, RecordId> tuple : randomMap(maxSegments, maxEntriesPerSegment).entrySet()) {
- reference.put(tuple.getKey(), tuple.getValue());
- map.put(tuple.getKey(), tuple.getValue());
- }
- }
-
- private void removeRandomEntries(int count) {
- Set<SegmentId> remove = newHashSet();
- for (int k = 0; k < count && !reference.isEmpty(); k++) {
- int j = rnd.nextInt(reference.size());
- remove.add(get(reference.keySet(), j).getSegmentId());
- }
-
- Set<UUID> removeUUIDs = newHashSet();
- for (SegmentId sid : remove) {
- removeUUIDs.add(new UUID(sid.getMostSignificantBits(), sid.getLeastSignificantBits()));
- Iterator<RecordId> it = reference.keySet().iterator();
- while (it.hasNext()) {
- if (sid.equals(it.next().getSegmentId())) {
- it.remove();
- }
- }
- }
-
- map.compress(removeUUIDs);
- }
-
- private void checkMap() {
- for (Entry<RecordId, RecordId> entry : reference.entrySet()) {
- assertTrue("Failed with seed " + SEED,
- map.wasCompactedTo(entry.getKey(), entry.getValue()));
- assertFalse("Failed with seed " + SEED,
- map.wasCompactedTo(entry.getValue(), entry.getKey()));
+ public CompactionMapTest(boolean usePersistedMap) {
+ SegmentWriter writer = usePersistedMap
+ ? new SegmentWriter(store, tracker, V_11)
+ : null;
+ compactionMap1 = createCompactionMap(tracker, writer);
+ referenceMap1 = randomRecordIdMap(rnd, tracker, 10, 10);
+ putAll(compactionMap1, referenceMap1);
+ referenceMap.putAll(referenceMap1);
+
+ compactionMap2 = createCompactionMap(tracker, writer);
+ referenceMap2 = randomRecordIdMap(rnd, tracker, 10, 10);
+ putAll(compactionMap2, referenceMap2);
+ referenceMap.putAll(referenceMap2);
+
+ compactionMap3 = createCompactionMap(tracker, writer);
+ referenceMap3 = randomRecordIdMap(rnd, tracker, 10, 10);
+ putAll(compactionMap3, referenceMap3);
+ referenceMap.putAll(referenceMap3);
+
+ this.compactionMap = CompactionMap.EMPTY.cons(compactionMap3).cons(compactionMap2).cons(compactionMap1);
+ }
+
+ private static void putAll(PartialCompactionMap map1, Map<RecordId, RecordId> recordIdRecordIdMap) {
+ for (Entry<RecordId, RecordId> tuple : recordIdRecordIdMap.entrySet()) {
+ map1.put(tuple.getKey(), tuple.getValue());
}
}
@Test
- public void randomTest() {
- int maxSegments = 10000;
- int maxEntriesPerSegment = 10;
-
- for (int k = 0; k < 10; k++) {
- addRandomEntries(rnd.nextInt(maxSegments) + 1, rnd.nextInt(maxEntriesPerSegment) + 1);
- if (!reference.isEmpty()) {
- removeRandomEntries(rnd.nextInt(reference.size()));
- }
- checkMap();
+ public void checkExistingKeys() {
+ for (Entry<RecordId, RecordId> reference : referenceMap.entrySet()) {
+ assertEquals(reference.getValue(), compactionMap.get((reference.getKey())));
}
- map.compress();
- checkMap();
}
@Test
- public void benchLargeMap() {
- assumeTrue(BENCH);
-
- // check the memory use of really large mappings, 1M compacted segments with 10 records each.
- Runtime runtime = Runtime.getRuntime();
- Stopwatch timer = Stopwatch.createStarted();
- for (int i = 0; i < 1000000; i++) {
- if (i % 100000 == 0) {
- System.gc();
- System.out.println(
- i + ": " + (runtime.totalMemory() - runtime.freeMemory()) /
- (1024 * 1024) + "MB, " + timer.toString());
- timer.reset();
- timer.start();
- }
- SegmentId sid = getTracker().newDataSegmentId();
- for (int j = 0; j < 10; j++) {
- RecordId rid = new RecordId(sid, j << RECORD_ALIGN_BITS);
- map.put(rid, rid);
+ public void checkNonExistingKeys() {
+ for (RecordId keys : randomRecordIdMap(rnd, tracker, 10, 10).keySet()) {
+ if (!referenceMap.containsKey(keys)) {
+ assertNull(compactionMap.get(keys));
}
}
- map.compress();
-
- System.gc();
- System.out.println(
- "final: " + (runtime.totalMemory() - runtime.freeMemory()) /
- (1024 * 1024) + "MB, " + timer.toString());
}
@Test
- public void benchPut() throws Exception {
- assumeTrue(BENCH);
-
- run(new PutBenchmark(0, 0));
- run(new PutBenchmark(1000, 10));
- run(new PutBenchmark(10000, 10));
- run(new PutBenchmark(100000, 10));
- run(new PutBenchmark(1000000, 10));
- }
-
- @Test
- public void benchGet() throws Exception {
- assumeTrue(BENCH);
-
- run(new GetBenchmark(1000, 10));
- run(new GetBenchmark(10000, 10));
- run(new GetBenchmark(100000, 10));
- run(new GetBenchmark(1000000, 10));
- }
-
- private class PutBenchmark extends Benchmark {
- private final int maxSegments;
- private final int maxEntriesPerSegment;
-
- private Map<RecordId, RecordId> putIds;
-
- public PutBenchmark(int maxSegments, int maxEntriesPerSegment) {
- this.maxSegments = maxSegments;
- this.maxEntriesPerSegment = maxEntriesPerSegment;
+ public void removeSome() {
+ Set<UUID> removedUUIDs = newHashSet();
+ for (int k = 0; k < 1 + rnd.nextInt(referenceMap.size()); k++) {
+ RecordId key = get(referenceMap.keySet(), rnd.nextInt(referenceMap.size()));
+ removedUUIDs.add(key.asUUID());
}
- @Override
- public void setup() throws IOException {
- if (maxSegments > 0) {
- addRandomEntries(maxSegments, maxEntriesPerSegment);
- }
- }
-
- @Override
- public void beforeRun() throws Exception {
- putIds = randomMap(1000, 10);
- }
+ compactionMap.remove(removedUUIDs);
- @Override
- public void run() {
- for (Entry<RecordId, RecordId> tuple : putIds.entrySet()) {
- map.put(tuple.getKey(), tuple.getValue());
+ for (Entry<RecordId, RecordId> reference : referenceMap.entrySet()) {
+ RecordId key = reference.getKey();
+ if (removedUUIDs.contains(key.asUUID())) {
+ assertNull(compactionMap.get(key));
+ } else {
+ assertEquals(reference.getValue(), compactionMap.get(key));
}
}
-
- @Override
- public String toString() {
- return "Put benchmark: maxSegments=" + maxSegments + ", maxEntriesPerSegment=" + maxEntriesPerSegment;
- }
}
- private class GetBenchmark extends Benchmark {
- private final int maxSegments;
- private final int maxEntriesPerSegment;
-
- private final List<RecordId> getCandidateIds = newArrayList();
- private final List<RecordId> getIds = newArrayList();
-
- public GetBenchmark(int maxSegments, int maxEntriesPerSegment) {
- this.maxSegments = maxSegments;
- this.maxEntriesPerSegment = maxEntriesPerSegment;
- }
-
- @Override
- public void setup() throws IOException {
- addRandomEntries(maxSegments, maxEntriesPerSegment);
- map.compress();
- for (RecordId recordId : reference.keySet()) {
- if (rnd.nextInt(reference.size()) % 10000 == 0) {
- getCandidateIds.add(recordId);
- }
- }
- for (int k = 0; k < 10000; k++) {
- getCandidateIds.add(new RecordId(
- getTracker().newDataSegmentId(),
- newValidOffset(rnd, 0, MAX_SEGMENT_SIZE)));
- }
- }
-
- @Override
- public void beforeRun() throws Exception {
- for (int k = 0; k < 10000; k ++) {
- getIds.add(getCandidateIds.get(rnd.nextInt(getCandidateIds.size())));
- }
- }
-
- @Override
- public void run() {
- for (RecordId id : getIds) {
- map.get(id);
- }
- }
-
- @Override
- public void afterRun() throws Exception {
- getIds.clear();
+ private static long countUUIDs(Set<RecordId> recordIds) {
+ Set<UUID> uuids = newHashSet();
+ for (RecordId recordId : recordIds) {
+ uuids.add(recordId.asUUID());
}
+ return uuids.size();
+ }
- @Override
- public String toString() {
- return "Get benchmark: maxSegments=" + maxSegments + ", maxEntriesPerSegment=" + maxEntriesPerSegment;
+ @Test
+ public void removeGeneration() {
+ compactionMap1.compress();
+ compactionMap2.compress();
+ compactionMap3.compress();
+
+ assertArrayEquals(new long[]{10, 10, 10}, compactionMap.getSegmentCounts());
+ assertArrayEquals(new long[] {100, 100, 100}, compactionMap.getRecordCounts());
+
+ int expectedDepth = 3;
+ long expectedSize = countUUIDs(referenceMap.keySet());
+ assertEquals(expectedDepth, compactionMap.getDepth());
+ assertEquals(expectedSize, sum(compactionMap.getSegmentCounts()));
+
+ for (Map<RecordId, RecordId> referenceMap : ImmutableList.of(referenceMap2, referenceMap1, referenceMap3)) {
+ Set<UUID> removedUUIDs = newHashSet();
+ for (RecordId key : referenceMap.keySet()) {
+ removedUUIDs.add(key.asUUID());
+ }
+ compactionMap.remove(removedUUIDs);
+ assertEquals(--expectedDepth, compactionMap.getDepth());
+ expectedSize -= removedUUIDs.size();
+ assertEquals(expectedSize, sum(compactionMap.getSegmentCounts()));
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/HeavyWriteIT.java Fri Jun 5 15:09:44 2015
@@ -28,6 +28,7 @@ import static org.junit.Assume.assumeTru
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
+import java.util.List;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -35,6 +36,7 @@ import java.util.concurrent.atomic.Atomi
import javax.annotation.Nonnull;
+import com.google.common.collect.ImmutableList;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.api.CommitFailedException;
import org.apache.jackrabbit.oak.commons.FixturesHelper;
@@ -49,11 +51,26 @@ import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+@RunWith(Parameterized.class)
public class HeavyWriteIT {
private static final Set<Fixture> FIXTURES = FixturesHelper.getFixtures();
+
+ private final boolean usePersistedMap;
+
private File directory;
+ @Parameterized.Parameters
+ public static List<Boolean[]> fixtures() {
+ return ImmutableList.of(new Boolean[] {true}, new Boolean[] {false});
+ }
+
+ public HeavyWriteIT(boolean usePersistedMap) {
+ this.usePersistedMap = usePersistedMap;
+ }
+
@BeforeClass
public static void checkFixtures() {
assumeTrue(!travisIntegrationTesting()); // FIXME OAK-2375. Often fails on Travis
@@ -77,13 +94,15 @@ public class HeavyWriteIT {
public void heavyWrite() throws IOException, CommitFailedException, InterruptedException {
final FileStore store = new FileStore(directory, 128, false);
final SegmentNodeStore nodeStore = new SegmentNodeStore(store);
- store.setCompactionStrategy(new CompactionStrategy(false, false,
+ CompactionStrategy custom = new CompactionStrategy(false, false,
CLEAN_OLD, 30000, (byte) 0) {
@Override
public boolean compacted(@Nonnull Callable<Boolean> setHead) throws Exception {
return nodeStore.locked(setHead);
}
- });
+ };
+ custom.setPersistCompactionMap(usePersistedMap);
+ store.setCompactionStrategy(custom);
int writes = 100;
final AtomicBoolean run = new AtomicBoolean(true);
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/PartialCompactionMapTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/PartialCompactionMapTest.java?rev=1683780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/PartialCompactionMapTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/PartialCompactionMapTest.java Fri Jun 5 15:09:44 2015
@@ -0,0 +1,378 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.collect.Iterables.get;
+import static com.google.common.collect.Lists.newArrayList;
+import static com.google.common.collect.Maps.newHashMap;
+import static com.google.inject.internal.util.$Sets.newHashSet;
+import static java.io.File.createTempFile;
+import static junit.framework.Assert.assertTrue;
+import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCount;
+import static org.apache.jackrabbit.oak.commons.benchmark.MicroBenchmark.run;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentVersion.V_11;
+import static org.apache.jackrabbit.oak.plugins.segment.TestUtils.newValidOffset;
+import static org.apache.jackrabbit.oak.plugins.segment.TestUtils.randomRecordIdMap;
+import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assume.assumeTrue;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+import java.util.Set;
+import java.util.UUID;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+import org.apache.jackrabbit.oak.commons.benchmark.MicroBenchmark.Benchmark;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class PartialCompactionMapTest {
+ private static final boolean BENCH = Boolean.getBoolean("benchmark");
+ private static final int SEED = Integer.getInteger("SEED", new Random().nextInt());
+
+ private final Random rnd = new Random(SEED);
+ private final boolean usePersistedMap;
+
+ private File directory;
+ private SegmentStore segmentStore;
+
+ private Map<RecordId, RecordId> reference;
+ private PartialCompactionMap map;
+
+ @Parameterized.Parameters
+ public static List<Boolean[]> fixtures() {
+ return ImmutableList.of(new Boolean[] {true}, new Boolean[] {false});
+ }
+
+ public PartialCompactionMapTest(boolean usePersistedMap) {
+ this.usePersistedMap = usePersistedMap;
+ }
+
+ @Before
+ public void setup() throws IOException {
+ directory = createTempFile(PartialCompactionMapTest.class.getSimpleName(), "dir", new File("target"));
+ directory.delete();
+ directory.mkdir();
+
+ segmentStore = newFileStore(directory).create();
+ }
+
+ @After
+ public void tearDown() {
+ segmentStore.close();
+ directory.delete();
+ }
+
+ private SegmentTracker getTracker() {
+ return segmentStore.getTracker();
+ }
+
+ private PartialCompactionMap createCompactionMap() {
+ SegmentWriter writer = new SegmentWriter(segmentStore, getTracker(), V_11);
+ if (usePersistedMap) {
+ return new PersistedCompactionMap(writer);
+ } else {
+ return new InMemoryCompactionMap(writer.getTracker());
+ }
+ }
+
+ private void addAll(Map<RecordId, RecordId> toAdd) {
+ assert map != null;
+ for (Entry<RecordId, RecordId> tuple : toAdd.entrySet()) {
+ if (reference != null) {
+ reference.put(tuple.getKey(), tuple.getValue());
+ }
+ map.put(tuple.getKey(), tuple.getValue());
+ }
+ }
+
+ private void addRandomEntries(int segmentCount, int entriesPerSegment) {
+ assert map != null;
+ for (int k = 0; k < segmentCount / 1000; k++) {
+ addAll(randomRecordIdMap(rnd, getTracker(), 1000, entriesPerSegment));
+ }
+ addAll(randomRecordIdMap(rnd, getTracker(), segmentCount % 1000, entriesPerSegment));
+ }
+
+ private void removeRandomEntries(int count) {
+ assert reference != null;
+ assert map != null;
+ Set<SegmentId> remove = newHashSet();
+ for (int k = 0; k < count && !reference.isEmpty(); k++) {
+ int j = rnd.nextInt(reference.size());
+ remove.add(get(reference.keySet(), j).getSegmentId());
+ }
+
+ Set<UUID> removeUUIDs = newHashSet();
+ for (SegmentId sid : remove) {
+ removeUUIDs.add(new UUID(sid.getMostSignificantBits(), sid.getLeastSignificantBits()));
+ Iterator<RecordId> it = reference.keySet().iterator();
+ while (it.hasNext()) {
+ if (sid.equals(it.next().getSegmentId())) {
+ it.remove();
+ }
+ }
+ }
+
+ map.remove(removeUUIDs);
+ }
+
+ private void checkMap() {
+ assert reference != null;
+ assert map != null;
+ for (Entry<RecordId, RecordId> entry : reference.entrySet()) {
+ assertTrue("Failed with seed " + SEED,
+ map.wasCompactedTo(entry.getKey(), entry.getValue()));
+ assertFalse("Failed with seed " + SEED,
+ map.wasCompactedTo(entry.getValue(), entry.getKey()));
+ }
+ }
+
+ @Test
+ public void single() {
+ map = createCompactionMap();
+ RecordId before = RecordId.fromString(getTracker(), "00000000-0000-0000-0000-000000000000.0000");
+ RecordId after = RecordId.fromString(getTracker(), "11111111-1111-1111-1111-111111111111.1111");
+
+ map.put(before, after);
+ assertEquals(after, map.get(before));
+ map.compress();
+ assertEquals(after, map.get(before));
+ assertEquals(1, map.getRecordCount());
+ assertEquals(1, map.getSegmentCount());
+ }
+
+ @Test
+ public void remove() {
+ map = createCompactionMap();
+ RecordId before1 = RecordId.fromString(getTracker(), "00000000-0000-0000-0000-000000000000.0000");
+ RecordId before2 = RecordId.fromString(getTracker(), "00000000-0000-0000-0000-000000000000.1111");
+ RecordId after1 = RecordId.fromString(getTracker(), "11111111-1111-1111-1111-111111111111.0000");
+ RecordId after2 = RecordId.fromString(getTracker(), "11111111-1111-1111-1111-111111111111.1111");
+
+ map.put(before1, after1);
+ map.compress();
+ map.put(before2, after2);
+ assertEquals(after1, map.get(before1));
+ assertEquals(after2, map.get(before2));
+
+ map.remove(Sets.newHashSet(before1.asUUID()));
+ assertNull(map.get(before1));
+ assertNull(map.get(before2));
+ assertEquals(0, map.getRecordCount());
+ assertEquals(0, map.getSegmentCount());
+ }
+
+ private static Set<UUID> toUUID(Set<RecordId> recordIds) {
+ Set<UUID> uuids = newHashSet();
+ for (RecordId recordId : recordIds) {
+ uuids.add(recordId.asUUID());
+ }
+ return uuids;
+ }
+
+ @Test
+ public void random() {
+ int maxSegments = 1000;
+ int entriesPerSegment = 10;
+ reference = newHashMap();
+ map = createCompactionMap();
+
+ for (int k = 0; k < 10; k++) {
+ addRandomEntries(rnd.nextInt(maxSegments) + 1, rnd.nextInt(entriesPerSegment) + 1);
+ if (!reference.isEmpty()) {
+ removeRandomEntries(rnd.nextInt(reference.size()));
+ }
+ checkMap();
+ }
+ map.compress();
+ assertEquals(reference.size(), map.getRecordCount());
+ assertEquals(toUUID(reference.keySet()).size(), map.getSegmentCount());
+ checkMap();
+ }
+
+ private static void assertHeapSize(long size) {
+ long mem = Runtime.getRuntime().maxMemory();
+ assertTrue("Need " + humanReadableByteCount(size) +
+ ", only found " + humanReadableByteCount(mem), mem >= size);
+ }
+
+ @Test
+ public void benchLargeMap() {
+ assumeTrue(BENCH);
+ assertHeapSize(1000000000);
+
+ map = createCompactionMap();
+
+ // check the memory use of really large mappings, 1M compacted segments with 10 records each.
+ Runtime runtime = Runtime.getRuntime();
+ for (int i = 0; i < 1000; i++) {
+ Map<RecordId, RecordId> ids = randomRecordIdMap(rnd, getTracker(), 10000, 100);
+ long start = System.nanoTime();
+ for (Entry<RecordId, RecordId> entry : ids.entrySet()) {
+ map.put(entry.getKey(), entry.getValue());
+ }
+ System.out.println(
+ (i + 1) + ": " + (runtime.totalMemory() - runtime.freeMemory()) /
+ (1024 * 1024) + "MB, " + (System.nanoTime() - start) / 1000000 + "ms");
+ }
+ }
+
+ @Test
+ public void benchPut() throws Exception {
+ assumeTrue(BENCH);
+ assertHeapSize(4000000000L);
+
+ run(new PutBenchmark(0, 100));
+ run(new PutBenchmark(10, 100));
+ run(new PutBenchmark(100, 100));
+ run(new PutBenchmark(1000, 100));
+ run(new PutBenchmark(10000, 100));
+ run(new PutBenchmark(100000, 100));
+ run(new PutBenchmark(1000000, 100));
+ }
+
+ @Test
+ public void benchGet() throws Exception {
+ assumeTrue(BENCH);
+ assertHeapSize(4000000000L);
+
+ run(new GetBenchmark(0, 100));
+ run(new GetBenchmark(10, 100));
+ run(new GetBenchmark(100, 100));
+ run(new GetBenchmark(1000, 100));
+ run(new GetBenchmark(10000, 100));
+ run(new GetBenchmark(100000, 100));
+ run(new GetBenchmark(1000000, 100));
+ }
+
+ private class PutBenchmark extends Benchmark {
+ private final int segmentCount;
+ private final int entriesPerSegment;
+
+ private Map<RecordId, RecordId> putIds;
+
+ public PutBenchmark(int segmentCount, int entriesPerSegment) {
+ this.segmentCount = segmentCount;
+ this.entriesPerSegment = entriesPerSegment;
+ }
+
+ @Override
+ public void setup() throws IOException {
+ map = createCompactionMap();
+ if (segmentCount > 0) {
+ addRandomEntries(segmentCount, entriesPerSegment);
+ }
+ }
+
+ @Override
+ public void beforeRun() throws Exception {
+ putIds = randomRecordIdMap(rnd, getTracker(), 10000 / entriesPerSegment, entriesPerSegment);
+ }
+
+ @Override
+ public void run() {
+ for (Entry<RecordId, RecordId> tuple : putIds.entrySet()) {
+ map.put(tuple.getKey(), tuple.getValue());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "Put benchmark: SegmentCount=" + segmentCount + ", entriesPerSegment=" + entriesPerSegment;
+ }
+ }
+
+ private class GetBenchmark extends Benchmark {
+ private final int segmentCount;
+ private final int entriesPerSegment;
+
+ private final List<RecordId> getCandidateIds = newArrayList();
+ private final List<RecordId> getIds = newArrayList();
+
+ public GetBenchmark(int segmentCount, int entriesPerSegment) {
+ this.segmentCount = segmentCount;
+ this.entriesPerSegment = entriesPerSegment;
+ }
+
+ @Override
+ public void setup() throws IOException {
+ map = createCompactionMap();
+ reference = new HashMap<RecordId, RecordId>() {
+ @Override
+ public RecordId put(RecordId key, RecordId value) {
+ // Wow, what a horrendous hack!!
+ if (key.getSegmentId().getMostSignificantBits() % 10000 == 0) {
+ getCandidateIds.add(key);
+ }
+ return null;
+ }
+ };
+
+ addRandomEntries(segmentCount, entriesPerSegment);
+ map.compress();
+ for (int k = 0; k < 10000; k++) {
+ getCandidateIds.add(new RecordId(
+ getTracker().newDataSegmentId(),
+ newValidOffset(rnd, 0, MAX_SEGMENT_SIZE)));
+ }
+ }
+
+ @Override
+ public void beforeRun() throws Exception {
+ for (int k = 0; k < 10000; k ++) {
+ getIds.add(getCandidateIds.get(rnd.nextInt(getCandidateIds.size())));
+ }
+ }
+
+ @Override
+ public void run() {
+ for (RecordId id : getIds) {
+ map.get(id);
+ }
+ }
+
+ @Override
+ public void afterRun() throws Exception {
+ getIds.clear();
+ }
+
+ @Override
+ public String toString() {
+ return "Get benchmark: segmentCount=" + segmentCount + ", entriesPerSegment=" + entriesPerSegment;
+ }
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordIdMapTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordIdMapTest.java?rev=1683780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordIdMapTest.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/RecordIdMapTest.java Fri Jun 5 15:09:44 2015
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ALIGN_BITS;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.encode;
+import static org.apache.jackrabbit.oak.plugins.segment.TestUtils.newValidOffset;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Random;
+
+import org.apache.jackrabbit.oak.plugins.segment.memory.MemoryStore;
+import org.junit.Test;
+
+public class RecordIdMapTest {
+
+ @Test
+ public void testRecordIdMap() {
+ int maxSegments = 1000;
+ int maxEntriesPerSegment = 10;
+ int seed = new Random().nextInt();
+ Random r = new Random(seed);
+
+ SegmentTracker tracker = new MemoryStore().getTracker();
+ RecordIdMap map = new RecordIdMap();
+ Map<Short, RecordId> reference = newHashMap();
+ int segments = r.nextInt(maxSegments);
+ for (int i = 0; i < segments; i++) {
+ SegmentId id = tracker.newDataSegmentId();
+ int n = r.nextInt(maxEntriesPerSegment);
+ int offset = MAX_SEGMENT_SIZE;
+ for (int j = 0; j < n; j++) {
+ offset = newValidOffset(r, (n - j) << RECORD_ALIGN_BITS, offset);
+ RecordId record = new RecordId(id, offset);
+ reference.put(encode(record.getOffset()), record);
+ }
+ }
+ for (Entry<Short, RecordId> entry : reference.entrySet()) {
+ map.put(entry.getKey(), entry.getValue());
+ }
+
+ assertEquals("Failed with seed " + seed, reference.size(), map.size());
+ for (Entry<Short, RecordId> entry : reference.entrySet()) {
+ short key = entry.getKey();
+ assertTrue("Failed with seed " + seed, map.containsKey(key));
+
+ RecordId expected = entry.getValue();
+ RecordId actual = map.get(key);
+ assertEquals("Failed with seed " + seed, expected, actual);
+ }
+
+ for (int k = 0; k < map.size(); k++) {
+ short key = map.getKey(k);
+ RecordId expected = reference.get(key);
+ RecordId actual = map.get(key);
+ assertEquals("Failed with seed " + seed, expected, actual);
+ assertEquals("Failed with seed " + seed, expected, map.getRecordId(k));
+ }
+ }
+}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionIT.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionIT.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionIT.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionIT.java Fri Jun 5 15:09:44 2015
@@ -33,6 +33,7 @@ import static java.util.concurrent.TimeU
import static java.util.concurrent.TimeUnit.SECONDS;
import static org.apache.commons.io.FileUtils.deleteDirectory;
import static org.apache.commons.lang.RandomStringUtils.randomAlphabetic;
+import static org.apache.jackrabbit.oak.plugins.segment.CompactionMap.sum;
import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.CleanupType.CLEAN_OLD;
import static org.apache.jackrabbit.oak.plugins.segment.compaction.CompactionStrategy.MEMORY_THRESHOLD_DEFAULT;
import static org.apache.jackrabbit.oak.plugins.segment.file.FileStore.newFileStore;
@@ -98,6 +99,8 @@ import org.slf4j.LoggerFactory;
* TODO Leverage longeivity test support from OAK-2771 once we have it.
*/
public class SegmentCompactionIT {
+ private static final boolean PERSIST_COMPACTION_MAP = Boolean.getBoolean("persist-compaction-map");
+
/** Only run if explicitly asked to via -Dtest=SegmentCompactionIT */
private static final boolean ENABLED =
SegmentCompactionIT.class.getSimpleName().equals(getProperty("test"));
@@ -218,11 +221,12 @@ public class SegmentCompactionIT {
fileStore = newFileStore(directory).withGCMonitor(gcMonitor).create();
nodeStore = new SegmentNodeStore(fileStore);
+ compactionStrategy.setPersistCompactionMap(PERSIST_COMPACTION_MAP);
fileStore.setCompactionStrategy(compactionStrategy);
}
@After
- public void tearDown() throws InterruptedException {
+ public void tearDown() {
try {
if (mBeanRegistration != null) {
mBeanRegistration.unregister();
@@ -469,7 +473,7 @@ public class SegmentCompactionIT {
return child;
}
- protected final PropertyState chooseRandomProperty(NodeState node) throws Exception {
+ protected final PropertyState chooseRandomProperty(NodeState node) {
int count = (int) node.getPropertyCount();
if (count > 0) {
return get(node.getProperties(), rnd.nextInt(count));
@@ -570,8 +574,8 @@ public class SegmentCompactionIT {
}
@Override
- public void compacted() {
- delegate.compacted();
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
+ delegate.compacted(segmentCounts, recordCounts, compactionMapWeights);
lastCompacted = System.currentTimeMillis();
}
@@ -715,6 +719,11 @@ public class SegmentCompactionIT {
}
@Override
+ public boolean getPersistCompactionMap() {
+ return compactionStrategy.getPersistCompactionMap();
+ }
+
+ @Override
public int getReaderCount() {
return readers.size();
}
@@ -739,14 +748,28 @@ public class SegmentCompactionIT {
}
}
+ private CompactionMap getCompactionMap() {
+ return fileStore.getTracker().getCompactionMap();
+ }
+
@Override
public long getCompactionMapWeight() {
- return fileStore.getTracker().getCompactionMap().getEstimatedWeight();
+ return sum(getCompactionMap().getEstimatedWeights());
+ }
+
+ @Override
+ public long getSegmentCount() {
+ return sum(getCompactionMap().getSegmentCounts());
+ }
+
+ @Override
+ public long getRecordCount() {
+ return sum(getCompactionMap().getRecordCounts());
}
@Override
public int getCompactionMapDepth() {
- return fileStore.getTracker().getCompactionMap().getDepth();
+ return getCompactionMap().getDepth();
}
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionMBean.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionMBean.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionMBean.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentCompactionMBean.java Fri Jun 5 15:09:44 2015
@@ -127,6 +127,12 @@ public interface SegmentCompactionMBean
boolean getRootReference();
/**
+ * Determine whether the compaction map is persisted or in memory
+ * @return {@code true} if persisted, {@code false} otherwise
+ */
+ boolean getPersistCompactionMap();
+
+ /**
* @return actual number of concurrent readers
*/
int getReaderCount();
@@ -152,6 +158,16 @@ public interface SegmentCompactionMBean
long getCompactionMapWeight();
/**
+ * @return number of record referenced by the keys in this map.
+ */
+ long getRecordCount();
+
+ /**
+ * @return number of segments referenced by the keys in this map.
+ */
+ long getSegmentCount();
+
+ /**
* @return current depth of the compaction map
*/
int getCompactionMapDepth();
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentDataStoreBlobGCTest.java Fri Jun 5 15:09:44 2015
@@ -35,9 +35,11 @@ import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nonnull;
+
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-
import com.google.common.util.concurrent.MoreExecutors;
import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.api.CommitFailedException;
@@ -56,22 +58,34 @@ import org.apache.jackrabbit.oak.spi.com
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.junit.After;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.annotation.Nonnull;
-
/**
* Tests for SegmentNodeStore DataStore GC
*/
+@RunWith(Parameterized.class)
public class SegmentDataStoreBlobGCTest {
private static final Logger log = LoggerFactory.getLogger(SegmentDataStoreBlobGCTest.class);
+ private final boolean usePersistedMap;
+
SegmentNodeStore nodeStore;
FileStore store;
DataStoreBlobStore blobStore;
Date startDate;
+ @Parameterized.Parameters
+ public static List<Boolean[]> fixtures() {
+ return ImmutableList.of(new Boolean[] {true}, new Boolean[] {false});
+ }
+
+ public SegmentDataStoreBlobGCTest(boolean usePersistedMap) {
+ this.usePersistedMap = usePersistedMap;
+ }
+
protected SegmentNodeStore getNodeStore(BlobStore blobStore) throws IOException {
if (nodeStore == null) {
store = new FileStore(blobStore, getWorkDir(), 256, false);
@@ -83,6 +97,7 @@ public class SegmentDataStoreBlobGCTest
return setHead.call();
}
};
+ compactionStrategy.setPersistCompactionMap(usePersistedMap);
store.setCompactionStrategy(compactionStrategy);
nodeStore = new SegmentNodeStore(store);
}
Added: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/TestUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/TestUtils.java?rev=1683780&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/TestUtils.java (added)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/TestUtils.java Fri Jun 5 15:09:44 2015
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.plugins.segment;
+
+import static com.google.common.collect.Maps.newHashMap;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.MAX_SEGMENT_SIZE;
+import static org.apache.jackrabbit.oak.plugins.segment.Segment.RECORD_ALIGN_BITS;
+
+import java.util.Map;
+import java.util.Random;
+
+import javax.annotation.Nonnull;
+
+public final class TestUtils {
+ private TestUtils() {}
+
+ // FIXME SegmentTestUtils duplicates this
+ /**
+ * Returns a new valid record offset, between {@code a} and {@code b},
+ * exclusive.
+ */
+ public static int newValidOffset(@Nonnull Random random, int a, int b) {
+ int p = (a >> RECORD_ALIGN_BITS) + 1;
+ int q = (b >> RECORD_ALIGN_BITS);
+ return (p + random.nextInt(q - p)) << RECORD_ALIGN_BITS;
+ }
+
+ /**
+ * Create a random map of record ids.
+ *
+ * @param rnd
+ * @param tracker
+ * @param segmentCount number of segments
+ * @param entriesPerSegment number of records per segment
+ * @return map of record ids
+ */
+ public static Map<RecordId, RecordId> randomRecordIdMap(Random rnd, SegmentTracker tracker,
+ int segmentCount, int entriesPerSegment) {
+ Map<RecordId, RecordId> map = newHashMap();
+ for (int i = 0; i < segmentCount; i++) {
+ SegmentId id = tracker.newDataSegmentId();
+ int offset = MAX_SEGMENT_SIZE;
+ for (int j = 0; j < entriesPerSegment; j++) {
+ offset = newValidOffset(rnd, (entriesPerSegment - j) << RECORD_ALIGN_BITS, offset);
+ RecordId before = new RecordId(id, offset);
+ RecordId after = new RecordId(
+ tracker.newDataSegmentId(),
+ newValidOffset(rnd, 0, MAX_SEGMENT_SIZE));
+ map.put(before, after);
+ }
+ }
+ return map;
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/file/CompactionEstimatorTest.java Fri Jun 5 15:09:44 2015
@@ -88,7 +88,7 @@ public class CompactionEstimatorTest {
try {
// should be at 66%
assertTrue(fileStore.estimateCompactionGain()
- .estimateCompactionGain() > 60);
+ .estimateCompactionGain(0) > 60);
} finally {
fileStore.close();
}
Modified: jackrabbit/oak/trunk/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java (original)
+++ jackrabbit/oak/trunk/oak-jcr/src/main/java/org/apache/jackrabbit/oak/jcr/repository/RepositoryImpl.java Fri Jun 5 15:09:44 2015
@@ -468,7 +468,7 @@ public class RepositoryImpl implements J
}
@Override
- public void compacted() {
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
compacted = true;
}
Modified: jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/RefreshOnGCTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/RefreshOnGCTest.java?rev=1683780&r1=1683779&r2=1683780&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/RefreshOnGCTest.java (original)
+++ jackrabbit/oak/trunk/oak-jcr/src/test/java/org/apache/jackrabbit/oak/jcr/RefreshOnGCTest.java Fri Jun 5 15:09:44 2015
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertTru
import java.io.File;
import java.io.IOException;
+import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
@@ -37,6 +38,7 @@ import javax.jcr.RepositoryException;
import javax.jcr.Session;
import javax.jcr.SimpleCredentials;
+import com.google.common.collect.ImmutableList;
import org.apache.jackrabbit.api.JackrabbitRepository;
import org.apache.jackrabbit.oak.Oak;
import org.apache.jackrabbit.oak.spi.gc.GCMonitorTracker;
@@ -49,12 +51,26 @@ import org.apache.jackrabbit.oak.spi.whi
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+@RunWith(Parameterized.class)
public class RefreshOnGCTest {
+ private final boolean usePersistedMap;
+
private FileStore fileStore;
private Repository repository;
private GCMonitorTracker gcMonitor;
+ @Parameterized.Parameters
+ public static List<Boolean[]> fixtures() {
+ return ImmutableList.of(new Boolean[] {true}, new Boolean[] {false});
+ }
+
+ public RefreshOnGCTest(boolean usePersistedMap) {
+ this.usePersistedMap = usePersistedMap;
+ }
+
@Before
public void setup() throws IOException {
File directory = createTempFile(getClass().getSimpleName(), "test", new File("target"));
@@ -64,17 +80,19 @@ public class RefreshOnGCTest {
Whiteboard whiteboard = new DefaultWhiteboard();
gcMonitor = new GCMonitorTracker();
gcMonitor.start(whiteboard);
+ CompactionStrategy strategy = new CompactionStrategy(
+ false, false, CLEAN_NONE, 0, CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) {
+ @Override
+ public boolean compacted(@Nonnull Callable<Boolean> setHead) throws Exception {
+ setHead.call();
+ return true;
+ }
+ };
+ strategy.setPersistCompactionMap(usePersistedMap);
fileStore = newFileStore(directory)
.withGCMonitor(gcMonitor)
.create()
- .setCompactionStrategy(new CompactionStrategy(
- false, false, CLEAN_NONE, 0, CompactionStrategy.MEMORY_THRESHOLD_DEFAULT) {
- @Override
- public boolean compacted(@Nonnull Callable<Boolean> setHead) throws Exception {
- setHead.call();
- return true;
- }
- });
+ .setCompactionStrategy(strategy);
NodeStore nodeStore = new SegmentNodeStore(fileStore);
Oak oak = new Oak(nodeStore);