You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2025/04/29 05:42:03 UTC
(jackrabbit-oak) 01/01: OAK-11687 DataStore FileCache Memory Usage
This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch OAK-11687
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 8f20de189f234d29f8fe19ecb75bf3f3c5fb06d0
Author: Thomas Mueller <th...@apache.org>
AuthorDate: Tue Apr 29 07:41:44 2025 +0200
OAK-11687 DataStore FileCache Memory Usage
---
.../jackrabbit/oak/plugins/blob/FileCache.java | 67 ++++++++++++----------
1 file changed, 38 insertions(+), 29 deletions(-)
diff --git a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
index 847d41d0f1..acb365da08 100644
--- a/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
+++ b/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
@@ -72,25 +72,16 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
*/
private File cacheRoot;
- private CacheLIRS<String, File> cache;
+ private CacheLIRS<String, String> cache;
private FileCacheStats cacheStats;
private ExecutorService executor;
- private CacheLoader<String, File> cacheLoader;
+ private CacheLoader<String, String> cacheLoader;
- /**
- * Convert the size calculation to KB to support max file size of 2 TB
- */
- private static final Weigher<String, File> weigher = (key, value) -> {
- // convert to number of 4 KB blocks
- return Math.round(value.length() / (4 * 1024));
- };
-
- //Rough estimate of the in-memory key, value pair
- private static final Weigher<String, File> memWeigher = (key, value) -> (StringUtils.estimateMemoryUsage(key) +
- StringUtils.estimateMemoryUsage(value.getAbsolutePath()) + 48);
+ private Weigher<String, String> weigher;
+ private Weigher<String, String> memWeigher;
private FileCache(long maxSize /* bytes */, File root,
final CacheLoader<String, InputStream> loader, @Nullable final ExecutorService executor) {
@@ -99,15 +90,27 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
this.cacheRoot = new File(root, DOWNLOAD_DIR);
// convert to number of 4 KB blocks
- long size = Math.round(maxSize / (1024L * 4));
+ long size = Math.round(maxSize / 4096);
+
+ /**
+ * Convert the size calculation to KB to support max file size of 2 TB
+ */
+ weigher = (key, value) -> {
+ // convert to number of 4 KB blocks, plus an overhead of 4 blocks per file
+ return Math.round(getFile(value).length() / 4096) + 4;
+ };
+
+ //Rough estimate of the in-memory key, value pair
+ memWeigher = (key, value) -> (StringUtils.estimateMemoryUsage(key) +
+ StringUtils.estimateMemoryUsage(value) + 48);
cacheLoader = new CacheLoader<>() {
@Override
- public File load(String key) throws Exception {
+ public String load(String key) throws Exception {
// Fetch from local cache directory and if not found load from backend
- File cachedFile = DataStoreCacheUtils.getFile(key, cacheRoot);
+ File cachedFile = getFile(key);
if (cachedFile.exists()) {
- return cachedFile;
+ return key;
} else {
long startNanos = System.nanoTime();
try (InputStream is = loader.load(key)) {
@@ -119,22 +122,22 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
if (LOG.isDebugEnabled()) {
LOG.debug("Loaded file: {} in {}", key, (System.nanoTime() - startNanos) / 1_000_000);
}
- return cachedFile;
+ return key;
}
}
};
- cache = new CacheLIRS.Builder<String, File>()
+ cache = new CacheLIRS.Builder<String, String>()
.maximumWeight(size)
.recordStats()
.weigher(weigher)
.segmentCount(SEGMENT_COUNT)
- .evictionCallback((key, cachedFile, cause) -> {
+ .evictionCallback((key, cachedKey, cause) -> {
try {
- if (cachedFile != null && cachedFile.exists()
+ if (cachedKey != null && getFile(cachedKey).exists()
&& cause != RemovalCause.REPLACED) {
- DataStoreCacheUtils.recursiveDelete(cachedFile, cacheRoot);
- LOG.info("File [{}] evicted with reason [{}]", cachedFile, cause);
+ DataStoreCacheUtils.recursiveDelete(getFile(cachedKey), cacheRoot);
+ LOG.info("File [{}] evicted with reason [{}]", getFile(cachedKey), cause);
}
} catch (IOException e) {
LOG.info("Cached file deletion failed after eviction", e);
@@ -158,6 +161,10 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
private FileCache() {
}
+ private File getFile(String key) {
+ return DataStoreCacheUtils.getFile(key, cacheRoot);
+ }
+
public static FileCache build(long maxSize /* bytes */, File root,
final CacheLoader<String, InputStream> loader, @Nullable final ExecutorService executor) {
if (maxSize > 0) {
@@ -186,7 +193,7 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
}
@Override public DataStoreCacheStatsMBean getStats() {
- return new FileCacheStats(cache, weigher, memWeigher, 0);
+ return new FileCacheStats(cache, (key, value) -> 1, (key, value) -> 1, 0);
}
@Override public void close() {
@@ -218,7 +225,7 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
FileUtils.moveFile(file, cached);
}
}
- cache.put(key, cached);
+ cache.put(key, key);
} catch (IOException e) {
LOG.error("Exception adding id [{}] with file [{}] to cache, root cause: {}", key, file, e.getMessage());
LOG.debug("Root cause", e);
@@ -238,7 +245,8 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
@Nullable
public File getIfPresent(String key) {
try {
- return cache.getIfPresent(key);
+ String cached = cache.getIfPresent(key);
+ return cached == null ? null : getFile(cached);
} catch (Exception e) {
LOG.error("Error in retrieving [{}] from cache", key, e);
}
@@ -254,7 +262,7 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
public File get(String key) throws IOException {
try {
// get from cache and download if not available
- return cache.get(key, () -> cacheLoader.load(key));
+ return getFile(cache.get(key, () -> cacheLoader.load(key)));
} catch (ExecutionException e) {
LOG.error("Error loading [{}] from cache", key);
throw new IOException(e);
@@ -319,7 +327,7 @@ public class FileCache extends AbstractCache<String, File> implements Closeable
}
class FileCacheStats extends CacheStats implements DataStoreCacheStatsMBean {
- private static final long KB = 4 * 1024;
+ private static final long BLOCK_SIZE = 4096;
private final Weigher<Object, Object> memWeigher;
private final Weigher<Object, Object> weigher;
private final Cache<Object, Object> cache;
@@ -361,8 +369,9 @@ class FileCacheStats extends CacheStats implements DataStoreCacheStatsMBean {
for (Map.Entry<?, ?> e : cache.asMap().entrySet()) {
Object k = e.getKey();
Object v = e.getValue();
- size += weigher.weigh(k, v) * KB;
+ size += weigher.weigh(k, v) * BLOCK_SIZE;
}
return size;
}
+
}