You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2016/07/26 21:13:23 UTC
svn commit: r1754180 - in
/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment:
RecordCache.java SegmentNodeStoreService.java WriterCacheManager.java
file/FileStoreBuilder.java
Author: mduerig
Date: Tue Jul 26 21:13:22 2016
New Revision: 1754180
URL: http://svn.apache.org/viewvc?rev=1754180&view=rev
Log:
OAK-4277: Finalise de-duplication caches
Configuration for string and template deduplication caches
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java?rev=1754180&r1=1754179&r2=1754180&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/RecordCache.java Tue Jul 26 21:13:22 2016
@@ -28,8 +28,6 @@ import javax.annotation.Nonnull;
import com.google.common.base.Supplier;
import com.google.common.cache.CacheStats;
-// FIXME OAK-4277: Finalise de-duplication caches
-// implement configuration
/**
* Partial mapping of keys of type {@code T} to values of type {@link RecordId}. This is
* typically used for de-duplicating values that have already been persisted and thus
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1754180&r1=1754179&r2=1754180&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java Tue Jul 26 21:13:22 2016
@@ -157,6 +157,20 @@ public class SegmentNodeStoreService ext
public static final String TEMPLATE_CACHE_SIZE = "templateCache.size";
@Property(
+ intValue = 15000,
+ label = "String deduplication cache size",
+ description = "Maximum number of strings to keep in the deduplication cache"
+ )
+ public static final String STRING_DEDUPLICATION_CACHE_SIZE = "stringDeduplicationCache.size";
+
+ @Property(
+ intValue = 3000,
+ label = "Template deduplication cache size",
+ description = "Maximum number of templates to keep in the deduplication cache"
+ )
+ public static final String TEMPLATE_DEDUPLICATION_CACHE_SIZE = "templateDeduplicationCache.size";
+
+ @Property(
byteValue = MEMORY_THRESHOLD_DEFAULT,
label = "Memory Multiplier",
description = "TarMK compaction available memory multiplier needed to run compaction"
@@ -344,6 +358,8 @@ public class SegmentNodeStoreService ext
.withSegmentCacheSize(getSegmentCacheSize())
.withStringCacheSize(getStringCacheSize())
.withTemplateCacheSize(getTemplateCacheSize())
+ .withStringDeduplicationCacheSize(getStringDeduplicationCacheSize())
+ .withTemplateDeduplicationCacheSize(getTemplateDeduplicationCacheSize())
.withMaxFileSize(getMaxFileSize())
.withMemoryMapping(getMode().equals("64"))
.withGCMonitor(gcMonitor)
@@ -619,6 +635,14 @@ public class SegmentNodeStoreService ext
return Integer.parseInt(getCacheSize(TEMPLATE_CACHE_SIZE));
}
+ private int getStringDeduplicationCacheSize() {
+ return Integer.parseInt(getCacheSize(STRING_DEDUPLICATION_CACHE_SIZE));
+ }
+
+ private int getTemplateDeduplicationCacheSize() {
+ return Integer.parseInt(getCacheSize(TEMPLATE_DEDUPLICATION_CACHE_SIZE));
+ }
+
private String getMaxFileSizeProperty() {
String size = property(SIZE);
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java?rev=1754180&r1=1754179&r2=1754180&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/WriterCacheManager.java Tue Jul 26 21:13:22 2016
@@ -49,10 +49,19 @@ import org.apache.jackrabbit.oak.api.jmx
* generations being reused.
*/
public abstract class WriterCacheManager {
- private static final int DEFAULT_STRING_CACHE_SIZE = getInteger(
+
+ /**
+ * Default size of the string cache.
+ * @see #getStringCache(int)
+ */
+ public static final int DEFAULT_STRING_CACHE_SIZE = getInteger(
"oak.tar.stringsCacheSize", 15000);
- private static final int DEFAULT_TEMPLATE_CACHE_SIZE = getInteger(
+ /**
+ * Default size of the template cache.
+ * @see #getTemplateCache(int)
+ */
+ public static final int DEFAULT_TEMPLATE_CACHE_SIZE = getInteger(
"oak.tar.templatesCacheSize", 3000);
/**
@@ -179,7 +188,8 @@ public abstract class WriterCacheManager
/**
* New instance using the default factories {@link RecordCache#factory(int)}
- * and {@link NodeCache#factory(int, int)}.
+ * and {@link NodeCache#factory(int, int)} with the sizes
+ * {@link #DEFAULT_STRING_CACHE_SIZE} and {@link #DEFAULT_TEMPLATE_CACHE_SIZE}.
*/
public Default() {
this(RecordCache.<String>factory(DEFAULT_STRING_CACHE_SIZE),
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1754180&r1=1754179&r2=1754180&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Tue Jul 26 21:13:22 2016
@@ -24,6 +24,8 @@ import static com.google.common.base.Pre
import static org.apache.jackrabbit.oak.segment.CachingSegmentReader.DEFAULT_STRING_CACHE_MB;
import static org.apache.jackrabbit.oak.segment.CachingSegmentReader.DEFAULT_TEMPLATE_CACHE_MB;
import static org.apache.jackrabbit.oak.segment.SegmentCache.DEFAULT_SEGMENT_CACHE_MB;
+import static org.apache.jackrabbit.oak.segment.WriterCacheManager.DEFAULT_STRING_CACHE_SIZE;
+import static org.apache.jackrabbit.oak.segment.WriterCacheManager.DEFAULT_TEMPLATE_CACHE_SIZE;
import static org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions.defaultGCOptions;
import java.io.File;
@@ -33,6 +35,9 @@ import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
import com.google.common.base.Predicate;
+import org.apache.jackrabbit.oak.segment.NodeCache;
+import org.apache.jackrabbit.oak.segment.RecordCache;
+import org.apache.jackrabbit.oak.segment.Template;
import org.apache.jackrabbit.oak.segment.WriterCacheManager;
import org.apache.jackrabbit.oak.segment.compaction.LoggingGCMonitor;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
@@ -64,10 +69,11 @@ public class FileStoreBuilder {
private int templateCacheSize = DEFAULT_TEMPLATE_CACHE_MB;
- private boolean memoryMapping;
+ private int stringDeduplicationCacheSize = DEFAULT_STRING_CACHE_SIZE;
- @Nonnull
- private final DelegatingGCMonitor gcMonitor = new DelegatingGCMonitor();
+ private int templateDeduplicationCacheSize = DEFAULT_TEMPLATE_CACHE_SIZE;
+
+ private boolean memoryMapping;
@Nonnull
private StatisticsProvider statsProvider = StatisticsProvider.NOOP;
@@ -76,70 +82,61 @@ public class FileStoreBuilder {
private SegmentGCOptions gcOptions = defaultGCOptions();
@Nonnull
- private GCListener gcListener;
+ private final EvictingWriteCacheManager cacheManager = new EvictingWriteCacheManager(
+ stringDeduplicationCacheSize, templateDeduplicationCacheSize);
@Nonnull
- private final WriterCacheManager cacheManager = new WriterCacheManager.Default() {{
- gcListener = new GCListener() {
- @Override
- public void info(String message, Object... arguments) {
- gcMonitor.info(message, arguments);
- }
-
- @Override
- public void warn(String message, Object... arguments) {
- gcMonitor.warn(message, arguments);
- }
-
- @Override
- public void error(String message, Exception exception) {
- gcMonitor.error(message, exception);
- }
-
- @Override
- public void skipped(String reason, Object... arguments) {
- gcMonitor.skipped(reason, arguments);
- }
-
- @Override
- public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
- gcMonitor.compacted(segmentCounts, recordCounts, compactionMapWeights);
- }
-
- @Override
- public void cleaned(long reclaimedSize, long currentSize) {
- gcMonitor.cleaned(reclaimedSize, currentSize);
- }
+ private final DelegatingGCMonitor gcMonitor = new DelegatingGCMonitor();
- @Override
- public void compacted(@Nonnull Status status, final int newGeneration) {
- switch (status) {
- case SUCCESS:
- // FIXME OAK-4283: Align GCMonitor API with implementation
- // This call is still needed to ensure upstream consumers
- // of GCMonitor callback get properly notified. See
- // RepositoryImpl.RefreshOnGC and
- // LuceneIndexProviderService.registerGCMonitor().
- gcMonitor.compacted(new long[0], new long[0], new long[0]);
- evictCaches(new Predicate<Integer>() {
- @Override
- public boolean apply(Integer generation) {
- return generation < newGeneration;
- }
- });
- break;
- case FAILURE:
- evictCaches(new Predicate<Integer>() {
- @Override
- public boolean apply(Integer generation) {
- return generation == newGeneration;
- }
- });
- break;
- }
+ @Nonnull
+ private final GCListener gcListener = new GCListener() {
+ @Override
+ public void info(String message, Object... arguments) {
+ gcMonitor.info(message, arguments);
+ }
+
+ @Override
+ public void warn(String message, Object... arguments) {
+ gcMonitor.warn(message, arguments);
+ }
+
+ @Override
+ public void error(String message, Exception exception) {
+ gcMonitor.error(message, exception);
+ }
+
+ @Override
+ public void skipped(String reason, Object... arguments) {
+ gcMonitor.skipped(reason, arguments);
+ }
+
+ @Override
+ public void compacted(long[] segmentCounts, long[] recordCounts, long[] compactionMapWeights) {
+ gcMonitor.compacted(segmentCounts, recordCounts, compactionMapWeights);
+ }
+
+ @Override
+ public void cleaned(long reclaimedSize, long currentSize) {
+ gcMonitor.cleaned(reclaimedSize, currentSize);
+ }
+
+ @Override
+ public void compacted(@Nonnull Status status, final int newGeneration) {
+ switch (status) {
+ case SUCCESS:
+ // FIXME OAK-4283: Align GCMonitor API with implementation
+ // This call is still needed to ensure upstream consumers
+ // of GCMonitor callback get properly notified. See
+ // RepositoryImpl.RefreshOnGC and
+ // LuceneIndexProviderService.registerGCMonitor().
+ gcMonitor.compacted(new long[0], new long[0], new long[0]);
+ cacheManager.evictOldGeneration(newGeneration);
+ break;
+ case FAILURE:
+ cacheManager.evictGeneration(newGeneration);
}
- };
- }};
+ }
+ };
@CheckForNull
private TarRevisions revisions;
@@ -215,6 +212,28 @@ public class FileStoreBuilder {
}
/**
+ * Number of items to keep in the string deduplication cache
+ * @param stringDeduplicationCacheSize None negative cache size
+ * @return this instance
+ */
+ @Nonnull
+ public FileStoreBuilder withStringDeduplicationCacheSize(int stringDeduplicationCacheSize) {
+ this.stringDeduplicationCacheSize = stringDeduplicationCacheSize;
+ return this;
+ }
+
+ /**
+ * Number of items to keep in the template deduplication cache
+ * @param templateDeduplicationCacheSize None negative cache size
+ * @return this instance
+ */
+ @Nonnull
+ public FileStoreBuilder withTemplateDeduplicationCacheSize(int templateDeduplicationCacheSize) {
+ this.templateDeduplicationCacheSize = templateDeduplicationCacheSize;
+ return this;
+ }
+
+ /**
* Turn memory mapping on or off
* @param memoryMapping
* @return this instance
@@ -287,6 +306,7 @@ public class FileStoreBuilder {
*/
@Nonnull
public FileStore build() throws IOException {
+ checkState(revisions == null, "Cannot re-use builder");
directory.mkdir();
revisions = new TarRevisions(false, directory);
LOG.info("Creating file store {}", this);
@@ -312,6 +332,7 @@ public class FileStoreBuilder {
*/
@Nonnull
public ReadOnlyStore buildReadOnly() throws IOException {
+ checkState(revisions == null, "Cannot re-use builder");
checkState(directory.exists() && directory.isDirectory());
revisions = new TarRevisions(true, directory);
LOG.info("Creating file store {}", this);
@@ -387,4 +408,30 @@ public class FileStoreBuilder {
", gcOptions=" + gcOptions +
'}';
}
+
+ private static class EvictingWriteCacheManager extends WriterCacheManager.Default {
+ public EvictingWriteCacheManager( int stringCacheSize, int templateCacheSize) {
+ super(RecordCache.<String>factory(stringCacheSize),
+ RecordCache.<Template>factory(templateCacheSize),
+ NodeCache.factory(1000000, 20));
+ }
+
+ void evictOldGeneration(final int newGeneration) {
+ evictCaches(new Predicate<Integer>() {
+ @Override
+ public boolean apply(Integer generation) {
+ return generation < newGeneration;
+ }
+ });
+ }
+
+ void evictGeneration(final int newGeneration) {
+ evictCaches(new Predicate<Integer>() {
+ @Override
+ public boolean apply(Integer generation) {
+ return generation == newGeneration;
+ }
+ });
+ }
+ }
}