You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2013/09/25 17:58:43 UTC
svn commit: r1526212 - in
/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk:
MongoMK.java NodeDocument.java
Author: mreutegg
Date: Wed Sep 25 15:58:43 2013
New Revision: 1526212
URL: http://svn.apache.org/r1526212
Log:
OAK-1034: OOME running Oak benchmark suite
- split documents when they reach 1MB
- cache names of child nodes up to 16k
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java?rev=1526212&r1=1526211&r2=1526212&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/MongoMK.java Wed Sep 25 15:58:43 2013
@@ -93,7 +93,7 @@ public class MongoMK implements MicroKer
* Do not cache more than this number of children for a document.
*/
private static final int NUM_CHILDREN_CACHE_LIMIT = Integer.getInteger(
- "oak.mongoMK.childrenCacheLimit", 1000);
+ "oak.mongoMK.childrenCacheLimit", 16 * 1024);
/**
* When trying to access revisions that are older than this many
@@ -617,7 +617,7 @@ public class MongoMK implements MicroKer
}
// check cache
NodeDocument.Children c = docChildrenCache.getIfPresent(path);
- if (c == null || (c.childNames.size() < limit && !c.isComplete)) {
+ if (c == null) {
c = new NodeDocument.Children();
List<NodeDocument> docs = store.query(Collection.NODES, from, to, limit);
for (NodeDocument doc : docs) {
@@ -626,6 +626,22 @@ public class MongoMK implements MicroKer
}
c.isComplete = docs.size() < limit;
docChildrenCache.put(path, c);
+ } else if (c.childNames.size() < limit && !c.isComplete) {
+ // fetch more and update cache
+ String lastName = c.childNames.get(c.childNames.size() - 1);
+ String lastPath = PathUtils.concat(path, lastName);
+ from = Utils.getIdFromPath(lastPath);
+ int remainingLimit = limit - c.childNames.size();
+ List<NodeDocument> docs = store.query(Collection.NODES,
+ from, to, remainingLimit);
+ NodeDocument.Children clone = c.clone();
+ for (NodeDocument doc : docs) {
+ String p = Utils.getPathFromId(doc.getId());
+ clone.childNames.add(PathUtils.getName(p));
+ }
+ clone.isComplete = docs.size() < remainingLimit;
+ docChildrenCache.put(path, clone);
+ c = clone;
}
return Iterables.transform(c.childNames, new Function<String, NodeDocument>() {
@Override
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java?rev=1526212&r1=1526211&r2=1526212&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java Wed Sep 25 15:58:43 2013
@@ -68,6 +68,12 @@ public class NodeDocument extends Docume
static final int SPLIT_CANDIDATE_THRESHOLD = 32 * 1024;
/**
+ * A document size threshold after which a split is forced even if
+ * {@link #REVISIONS_SPLIT_OFF_SIZE} is not reached.
+ */
+ static final int FORCE_SPLIT_THRESHOLD = 1024 * 1024;
+
+ /**
* Only split off at least this number of revisions.
*/
static final int REVISIONS_SPLIT_OFF_SIZE = 1000;
@@ -535,8 +541,10 @@ public class NodeDocument extends Docume
*/
@Nonnull
public Iterable<UpdateOp> split(@Nonnull RevisionContext context) {
- // only consider if there are enough commits
- if (getLocalRevisions().size() + getLocalCommitRoot().size() <= REVISIONS_SPLIT_OFF_SIZE) {
+ // only consider if there are enough commits,
+ // unless document is really big
+ if (getLocalRevisions().size() + getLocalCommitRoot().size() <= REVISIONS_SPLIT_OFF_SIZE
+ && getMemory() < FORCE_SPLIT_THRESHOLD) {
return Collections.emptyList();
}
String id = getId();
@@ -599,7 +607,8 @@ public class NodeDocument extends Docume
}
numValues += splitMap.size();
}
- if (high != null && low != null && numValues >= REVISIONS_SPLIT_OFF_SIZE) {
+ if (high != null && low != null
+ && (numValues >= REVISIONS_SPLIT_OFF_SIZE || getMemory() > FORCE_SPLIT_THRESHOLD)) {
// enough revisions to split off
splitOps = new ArrayList<UpdateOp>(2);
// move to another document
@@ -963,12 +972,12 @@ public class NodeDocument extends Docume
* The list of children for a node. The list might be complete or not, in
* which case it only represents a block of children.
*/
- static final class Children implements CacheValue {
+ static final class Children implements CacheValue, Cloneable {
/**
* The child node names, ordered as stored in MongoDB.
*/
- final List<String> childNames = new ArrayList<String>();
+ ArrayList<String> childNames = new ArrayList<String>();
/**
* Whether the list is complete (in which case there are no other
@@ -984,6 +993,18 @@ public class NodeDocument extends Docume
}
return size;
}
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Children clone() {
+ try {
+ Children clone = (Children) super.clone();
+ clone.childNames = (ArrayList<String>) childNames.clone();
+ return clone;
+ } catch (CloneNotSupportedException e) {
+ throw new RuntimeException();
+ }
+ }
}
/**