You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2020/03/13 12:39:49 UTC
svn commit: r1875151 - in /jackrabbit/oak/trunk:
oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/
oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/
oak-core-spi/src/test/java/org/apache/jackrabbit/oak/cache/
Author: thomasm
Date: Fri Mar 13 12:39:48 2020
New Revision: 1875151
URL: http://svn.apache.org/viewvc?rev=1875151&view=rev
Log:
OAK-8950 DataStore: FileCache should use one cache segment
Added:
jackrabbit/oak/trunk/oak-core-spi/src/test/java/org/apache/jackrabbit/oak/cache/CacheSizeTest.java
Modified:
jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileCacheTest.java
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java?rev=1875151&r1=1875150&r2=1875151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/main/java/org/apache/jackrabbit/oak/plugins/blob/FileCache.java Fri Mar 13 12:39:48 2020
@@ -61,6 +61,8 @@ public class FileCache extends AbstractC
*/
private static final Logger LOG = LoggerFactory.getLogger(FileCache.class);
+ private static final int SEGMENT_COUNT = Integer.getInteger("oak.blob.fileCache.segmentCount", 1);
+
protected static final String DOWNLOAD_DIR = "download";
/**
@@ -86,7 +88,8 @@ public class FileCache extends AbstractC
*/
private static final Weigher<String, File> weigher = new Weigher<String, File>() {
@Override public int weigh(String key, File value) {
- return Math.round(value.length() / (4 * 1024)); // convert to KB
+ // convert to number of 4 KB blocks
+ return Math.round(value.length() / (4 * 1024));
}};
//Rough estimate of the in-memory key, value pair
@@ -102,7 +105,7 @@ public class FileCache extends AbstractC
this.parent = root;
this.cacheRoot = new File(root, DOWNLOAD_DIR);
- /* convert to 4 KB block */
+ // convert to number of 4 KB blocks
long size = Math.round(maxSize / (1024L * 4));
cacheLoader = new CacheLoader<String, File>() {
@@ -133,6 +136,7 @@ public class FileCache extends AbstractC
.maximumWeight(size)
.recordStats()
.weigher(weigher)
+ .segmentCount(SEGMENT_COUNT)
.evictionCallback(new EvictionCallback<String, File>() {
@Override
public void evicted(@NotNull String key, @Nullable File cachedFile,
Modified: jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileCacheTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileCacheTest.java?rev=1875151&r1=1875150&r2=1875151&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileCacheTest.java (original)
+++ jackrabbit/oak/trunk/oak-blob-plugins/src/test/java/org/apache/jackrabbit/oak/plugins/blob/FileCacheTest.java Fri Mar 13 12:39:48 2020
@@ -223,6 +223,9 @@ public class FileCacheTest extends Abstr
@Test
public void getDifferentConcurrent() throws Exception {
LOG.info("Started getDifferentConcurrent");
+
+ cache = FileCache.build(4 * 1024/* KB */, root, loader, null);
+ closer.register(cache);
File f = createFile(0, loader, cache, folder);
File f2 = createFile(1, loader, cache, folder);
@@ -317,6 +320,9 @@ public class FileCacheTest extends Abstr
public void evictImplicit() throws Exception {
LOG.info("Started evictImplicit");
+ cache = FileCache.build(60 * 1024/* KB */, root, loader, null);
+ closer.register(cache);
+
for (int i = 0; i < 15; i++) {
File f = createFile(i, loader, cache, folder);
assertCache(i, cache, f);
@@ -324,8 +330,8 @@ public class FileCacheTest extends Abstr
File f = createFile(30, loader, cache, folder);
assertCache(30, cache, f);
- // One of the entries should have been evicted
- assertTrue(cache.getStats().getElementCount() == 15);
+ // Some entries should have been evicted
+ assertEquals(15, cache.getStats().getElementCount(), cache.getStats().getElementCount());
assertCacheStats(cache, 15, 60 * 1024, 16, 16);
LOG.info("Finished evictImplicit");
Added: jackrabbit/oak/trunk/oak-core-spi/src/test/java/org/apache/jackrabbit/oak/cache/CacheSizeTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core-spi/src/test/java/org/apache/jackrabbit/oak/cache/CacheSizeTest.java?rev=1875151&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-core-spi/src/test/java/org/apache/jackrabbit/oak/cache/CacheSizeTest.java (added)
+++ jackrabbit/oak/trunk/oak-core-spi/src/test/java/org/apache/jackrabbit/oak/cache/CacheSizeTest.java Fri Mar 13 12:39:48 2020
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+package org.apache.jackrabbit.oak.cache;
+
+import java.util.HashMap;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.jackrabbit.oak.cache.CacheLIRS.EvictionCallback;
+import org.junit.Test;
+
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.RemovalCause;
+import com.google.common.cache.Weigher;
+
+/**
+ * Test the maximum cache size (for the FileCache).
+ */
+public class CacheSizeTest {
+
+ private static final Weigher<String, FileObj> weigher = new Weigher<String, FileObj>() {
+ @Override
+ public int weigh(String key, FileObj value) {
+ return Math.round(value.length() / (4 * 1024)); // convert to 4 KB blocks
+ }
+ };
+
+ private HashMap<String, FileObj> files = new HashMap<>();
+
+ @Test
+ public void test() throws ExecutionException {
+
+ long maxSize = 20_000_000_000L;
+
+ long size = Math.round(maxSize / (1024L * 4));
+
+ CacheLoader<String, FileObj> cacheLoader = new CacheLoader<String, FileObj>() {
+
+ @Override
+ public FileObj load(String key) throws Exception {
+ // Fetch from local cache directory and if not found load from
+ // backend
+ FileObj cachedFile = getFile(key);
+ if (cachedFile.exists()) {
+ return cachedFile;
+ } else {
+ return loadFile(key);
+ }
+ }
+ };
+
+ CacheLIRS<String, FileObj> cache = new CacheLIRS.Builder<String, FileObj>().
+ maximumWeight(size).
+ recordStats().
+ weigher(weigher).
+ segmentCount(1).
+ evictionCallback(new EvictionCallback<String, FileObj>() {
+ @Override
+ public void evicted(String key, FileObj cachedFile, RemovalCause cause) {
+ if (cachedFile != null && cachedFile.exists() && cause != RemovalCause.REPLACED) {
+ delete(cachedFile);
+ }
+ }
+ }).build();
+
+ for (int i = 0; i < 15; i++) {
+ String name = "n" + i;
+ long length = 1_000_000_000;
+ files.put(name, new FileObj(name, length));
+ }
+ for (int i = 0; i < 100; i++) {
+ String name = "n" + i;
+ cache.get(name, () -> cacheLoader.load(name));
+ }
+
+ }
+
+ public FileObj getFile(String key) {
+ FileObj obj = files.get(key);
+ if (obj == null) {
+ // doesn't exist
+ return new FileObj(key, 0);
+ }
+ return obj;
+ }
+
+ public FileObj loadFile(String key) {
+ FileObj f = new FileObj(key, 10_000_000);
+ files.put(key, f);
+ return f;
+ }
+
+ private void delete(FileObj cachedFile) {
+ FileObj old = files.remove(cachedFile.getName());
+ if (old == null) {
+ throw new AssertionError("trying to remove a file that doesn't exist: " + cachedFile);
+ }
+ long totalLength = getDirectoryLength();
+ throw new AssertionError("removing a file: unexpected in this test; total length " + totalLength + " " + old);
+ }
+
+ private long getDirectoryLength() {
+ long length = 0;
+ for(FileObj obj : files.values()) {
+ length += obj.length;
+ }
+ return length;
+ }
+
+ class FileObj {
+
+ private final String name;
+ private final long length;
+
+ public FileObj(String name, long length) {
+ this.name = name;
+ this.length = length;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean exists() {
+ return files.containsKey(name);
+ }
+
+ public long length() {
+ return length;
+ }
+
+ public String toString() {
+ return name + "/" + length;
+ }
+
+ }
+}