You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by ch...@apache.org on 2013/06/10 15:18:06 UTC
svn commit: r1491442 - in /jackrabbit/oak/trunk/oak-mongomk/src:
main/java/org/apache/jackrabbit/mongomk/
main/java/org/apache/jackrabbit/mongomk/osgi/
main/java/org/apache/jackrabbit/mongomk/util/
main/resources/OSGI-INF/metatype/ test/java/org/apache...
Author: chetanm
Date: Mon Jun 10 13:18:05 2013
New Revision: 1491442
URL: http://svn.apache.org/r1491442
Log:
OAK-854 - Use weight based caching limit instead of size based limits
Adding support for using a Weigher which determines object weight through its size. The size determination logic uses emperical data to determine the size. Following changes done
- Exposed a config for cache size. Defaults to 256 MB
- The cache size is then distributed between three caches Node, Children and Document in 20%, 10%, 70% respectively
- Enabled stats collection by default. Later would make it configurable
Added:
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java (with props)
Modified:
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java
jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java
Added: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java?rev=1491442&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java (added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java Mon Jun 10 13:18:05 2013
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.mongomk;
+
+import com.google.common.cache.Weigher;
+import org.apache.jackrabbit.mongomk.util.Utils;
+
+/**
+ * Determines the weight of object based on the memory taken by them. The memory esimates
+ * are based on emperical data and not exact
+ */
+public class EmpericalWeigher implements Weigher<String,Object>{
+
+ @Override
+ public int weigh(String key, Object value) {
+ int size = key.length() * 2;
+
+ if(value instanceof Node){
+ size += ((Node) value).getMemory();
+ }else if (value instanceof Node.Children){
+ size += ((Node.Children) value).getMemory();
+ }else if(value instanceof MongoDocumentStore.CachedDocument){
+ size += Utils.estimateMemoryUsage(((MongoDocumentStore.CachedDocument) value).value);
+ }else if(value != null){
+ throw new IllegalArgumentException("Cannot determine weight for object of type " + value.getClass());
+ }
+ return size;
+ }
+}
Propchange: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/EmpericalWeigher.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoDocumentStore.java Mon Jun 10 13:18:05 2013
@@ -52,12 +52,6 @@ public class MongoDocumentStore implemen
private static final Logger LOG = LoggerFactory.getLogger(MongoDocumentStore.class);
- /**
- * The number of documents to cache.
- */
- private static final int CACHE_DOCUMENTS = Integer.getInteger(
- "oak.mongoMK.cacheDocs", 10 * 1024);
-
private static final boolean LOG_TIME = false;
private final DBCollection nodes;
@@ -70,7 +64,7 @@ public class MongoDocumentStore implemen
private final Cache<String, CachedDocument> nodesCache;
- public MongoDocumentStore(DB db) {
+ public MongoDocumentStore(DB db,MongoMK.Builder builder) {
nodes = db.getCollection(
Collection.NODES.toString());
clusterNodes = db.getCollection(
@@ -87,7 +81,9 @@ public class MongoDocumentStore implemen
// TODO expire entries if the parent was changed
nodesCache = CacheBuilder.newBuilder()
- .maximumSize(CACHE_DOCUMENTS)
+ .weigher(builder.getWeigher())
+ .recordStats()
+ .maximumWeight(builder.getDocumentCacheSize())
.build();
}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/MongoMK.java Mon Jun 10 13:18:05 2013
@@ -38,6 +38,7 @@ import javax.annotation.Nullable;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Weigher;
import org.apache.jackrabbit.mk.api.MicroKernel;
import org.apache.jackrabbit.mk.api.MicroKernelException;
import org.apache.jackrabbit.mk.blobs.BlobStore;
@@ -66,18 +67,6 @@ public class MongoMK implements MicroKer
private static final Logger LOG = LoggerFactory.getLogger(MongoMK.class);
/**
- * The number of child node list entries to cache.
- */
- private static final int CACHE_CHILDREN =
- Integer.getInteger("oak.mongoMK.cacheChildren", 1024);
-
- /**
- * The number of nodes to cache.
- */
- private static final int CACHE_NODES =
- Integer.getInteger("oak.mongoMK.cacheNodes", 5 * 1024);
-
- /**
* The number of diffs to cache.
*/
private static final int CACHE_DIFF =
@@ -231,13 +220,19 @@ public class MongoMK implements MicroKer
this.asyncDelay = builder.getAsyncDelay();
this.branches = new UnmergedBranches(revisionComparator);
- //TODO Use size based weigher
+ //TODO Make stats collection configurable as it add slight overhead
+ //TODO Expose the stats as JMX beans
+
nodeCache = CacheBuilder.newBuilder()
- .maximumSize(CACHE_NODES)
+ .weigher(builder.getWeigher())
+ .maximumWeight(builder.getNodeCacheSize())
+ .recordStats()
.build();
nodeChildrenCache = CacheBuilder.newBuilder()
- .maximumSize(CACHE_CHILDREN)
+ .weigher(builder.getWeigher())
+ .recordStats()
+ .maximumWeight(builder.getChildrenCacheSize())
.build();
diffCache = CacheBuilder.newBuilder()
@@ -1547,12 +1542,20 @@ public class MongoMK implements MicroKer
* A builder for a MongoMK instance.
*/
public static class Builder {
-
+ private static final long DEFAULT_MEMORY_CACHE_SIZE = 1 << 28; // 256MB
private DocumentStore documentStore;
private BlobStore blobStore;
private int clusterId = Integer.getInteger("oak.mongoMK.clusterId", 0);
private int asyncDelay = 1000;
private boolean timing;
+ private Weigher<String,Object> weigher = new EmpericalWeigher();
+ private long nodeCacheSize;
+ private long childrenCacheSize;
+ private long documentCacheSize;
+
+ public Builder() {
+ memoryCacheSize(DEFAULT_MEMORY_CACHE_SIZE);
+ }
/**
* Set the MongoDB connection to use. By default an in-memory store is used.
@@ -1562,7 +1565,7 @@ public class MongoMK implements MicroKer
*/
public Builder setMongoDB(DB db) {
if (db != null) {
- this.documentStore = new MongoDocumentStore(db);
+ this.documentStore = new MongoDocumentStore(db,this);
this.blobStore = new MongoBlobStore(db);
}
return this;
@@ -1650,7 +1653,35 @@ public class MongoMK implements MicroKer
public int getAsyncDelay() {
return asyncDelay;
}
-
+
+ public Weigher<String, Object> getWeigher() {
+ return weigher;
+ }
+
+ public Builder weigher(Weigher<String, Object> weigher) {
+ this.weigher = weigher;
+ return this;
+ }
+
+ public Builder memoryCacheSize(long memoryCacheSize) {
+ this.nodeCacheSize = memoryCacheSize * 20 / 100;
+ this.childrenCacheSize = memoryCacheSize * 10 / 100;
+ this.documentCacheSize = memoryCacheSize - nodeCacheSize - childrenCacheSize;
+ return this;
+ }
+
+ public long getNodeCacheSize() {
+ return nodeCacheSize;
+ }
+
+ public long getChildrenCacheSize() {
+ return childrenCacheSize;
+ }
+
+ public long getDocumentCacheSize() {
+ return documentCacheSize;
+ }
+
/**
* Open the MongoMK instance using the configured options.
*
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/Node.java Mon Jun 10 13:18:05 2013
@@ -126,6 +126,15 @@ public class Node {
this.path = path;
this.rev = rev;
}
+
+ public int getMemory(){
+ int size = 200 + path.length() * 2;
+ for(String c : children){
+ size += c.length() * 2 + 40;
+ }
+ return size;
+ }
+
@Override
public String toString() {
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/osgi/MongoMicroKernelService.java Mon Jun 10 13:18:05 2013
@@ -51,6 +51,7 @@ public class MongoMicroKernelService {
private static final String DEFAULT_HOST = "localhost";
private static final int DEFAULT_PORT = 27017;
private static final String DEFAULT_DB = "oak";
+ private static final int DEFAULT_CACHE = 256;
@Property(value = DEFAULT_HOST)
private static final String PROP_HOST = "host";
@@ -61,6 +62,10 @@ public class MongoMicroKernelService {
@Property(value = DEFAULT_DB)
private static final String PROP_DB = "db";
+ @Property(intValue=DEFAULT_CACHE)
+ private static final String PROP_CACHE = "cache";
+ private static final long MB = 1024 * 1024;
+
private final Logger logger = LoggerFactory.getLogger(this.getClass());
private ServiceRegistration reg;
@@ -72,6 +77,7 @@ public class MongoMicroKernelService {
String host = PropertiesUtil.toString(config.get(PROP_HOST), DEFAULT_HOST);
int port = PropertiesUtil.toInteger(config.get(PROP_PORT), DEFAULT_PORT);
String db = PropertiesUtil.toString(config.get(PROP_DB), DEFAULT_DB);
+ int cacheSize = PropertiesUtil.toInteger(config.get(PROP_CACHE), DEFAULT_CACHE);
logger.info("Starting MongoDB MicroKernel with host={}, port={}, db={}",
new Object[] {host, port, db});
@@ -81,7 +87,7 @@ public class MongoMicroKernelService {
logger.info("Connected to database {}", mongoDB);
- mk = new MongoMK.Builder().setMongoDB(mongoDB).open();
+ mk = new MongoMK.Builder().memoryCacheSize(cacheSize * MB).setMongoDB(mongoDB).open();
Properties props = new Properties();
props.setProperty("oak.mk.type", "mongo");
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/util/Utils.java Mon Jun 10 13:18:05 2013
@@ -22,6 +22,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
+import com.mongodb.BasicDBObject;
import org.bson.types.ObjectId;
/**
@@ -56,17 +57,26 @@ public class Utils {
return 0;
}
int size = 0;
+
for (Entry<String, Object> e : map.entrySet()) {
- size += e.getKey().length();
+ size += e.getKey().length()*2;
Object o = e.getValue();
if (o instanceof String) {
- size += o.toString().length();
+ size += ((String)o).length()*2;
} else if (o instanceof Long) {
size += 8;
+ } else if (o instanceof Integer) {
+ size += 4;
} else if (o instanceof Map) {
size += 8 + estimateMemoryUsage((Map<String, Object>) o);
}
}
+
+ if(map instanceof BasicDBObject){
+ //Based on emperical testing using JAMM
+ size += 176;
+ size += map.entrySet().size() * 136;
+ }
return size;
}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/resources/OSGI-INF/metatype/metatype.properties Mon Jun 10 13:18:05 2013
@@ -29,3 +29,6 @@ port.description = The port to connect t
db.name = MongoDB Database
db.description = The database to use.
+
+cache.name = Cache Size (MB)
+cache.description = Cache Size (MB)
\ No newline at end of file
Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MeasureMemory.java Mon Jun 10 13:18:05 2013
@@ -16,11 +16,14 @@
*/
package org.apache.jackrabbit.mongomk;
+import static org.apache.jackrabbit.mongomk.Node.Children;
import static org.junit.Assert.fail;
import java.util.LinkedList;
import java.util.concurrent.Callable;
+import com.mongodb.BasicDBObject;
+import org.apache.jackrabbit.mongomk.util.Utils;
import org.junit.Test;
/**
@@ -64,6 +67,50 @@ public class MeasureMemory {
}
});
}
+
+ @Test
+ public void basicObject() throws Exception {
+ measureMemory(new Callable<Object[]>() {
+ @Override
+ public Object[] call() {
+ BasicDBObject n = generateBasicObject(15);
+ return new Object[]{n, Utils.estimateMemoryUsage(n) + OVERHEAD};
+ }
+ });
+ }
+
+ @Test
+ public void basicObjectWithoutProperties() throws Exception {
+ measureMemory(new Callable<Object[]>() {
+ @Override
+ public Object[] call() {
+ BasicDBObject n = generateBasicObject(0);
+ return new Object[]{n, Utils.estimateMemoryUsage(n) + OVERHEAD};
+ }
+ });
+ }
+
+ @Test
+ public void nodeChild() throws Exception {
+ measureMemory(new Callable<Object[]>() {
+ @Override
+ public Object[] call() {
+ Children n = generateNodeChild(15);
+ return new Object[]{n, n.getMemory() + OVERHEAD};
+ }
+ });
+ }
+
+ @Test
+ public void nodeChildWithoutChildren() throws Exception {
+ measureMemory(new Callable<Object[]>() {
+ @Override
+ public Object[] call() {
+ Children n = generateNodeChild(15);
+ return new Object[]{n, n.getMemory() + OVERHEAD};
+ }
+ });
+ }
private static void measureMemory(Callable<Object[]> c) throws Exception {
LinkedList<Object> list = new LinkedList<Object>();
@@ -99,6 +146,22 @@ public class MeasureMemory {
n.setLastRevision(new Revision(1, 2, 3));
return n;
}
+
+ static BasicDBObject generateBasicObject(int propertyCount) {
+ BasicDBObject n = new BasicDBObject(new String("_id"),new String("/hello/world"));
+ for (int i = 0; i < propertyCount; i++) {
+ n.append("property" + i, "values " + i);
+ }
+ return n;
+ }
+
+ static Children generateNodeChild(int childCount) {
+ Children n = new Children(new String("_id"),new Revision(1, 2, 3));
+ for (int i = 0; i < childCount; i++) {
+ n.children.add("child"+i);
+ }
+ return n;
+ }
private static long getMemoryUsed() {
for (int i = 0; i < 10; i++) {
Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java?rev=1491442&r1=1491441&r2=1491442&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/MongoDocumentStoreTest.java Mon Jun 10 13:18:05 2013
@@ -54,7 +54,7 @@ public class MongoDocumentStoreTest {
DocumentStore openDocumentStore() {
if (MONGO_DB) {
- return new MongoDocumentStore(MongoUtils.getConnection().getDB());
+ return new MongoDocumentStore(MongoUtils.getConnection().getDB(),new MongoMK.Builder());
}
return new MemoryDocumentStore();
}