You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mr...@apache.org on 2013/09/16 11:16:20 UTC
svn commit: r1523568 - in /jackrabbit/oak/trunk/oak-core/src:
main/java/org/apache/jackrabbit/oak/plugins/mongomk/
test/java/org/apache/jackrabbit/oak/plugins/mongomk/
Author: mreutegg
Date: Mon Sep 16 09:16:19 2013
New Revision: 1523568
URL: http://svn.apache.org/r1523568
Log:
OAK-926: MongoMK: split documents when they are too large
- Split off DELETED and COMMIT_ROOT as well
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/Commit.java Mon Sep 16 09:16:19 2013
@@ -244,8 +244,11 @@ public class Commit {
}
}
for (UpdateOp op : changedNodes) {
- // set commit root on changed nodes
- NodeDocument.setCommitRoot(op, revision, commitRootDepth);
+ // set commit root on changed nodes unless it's the
+ // commit root itself
+ if (op != commitRoot) {
+ NodeDocument.setCommitRoot(op, revision, commitRootDepth);
+ }
opLog.add(op);
createOrUpdateNode(store, op);
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/NodeDocument.java Mon Sep 16 09:16:19 2013
@@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.plugin
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -150,19 +151,19 @@ public class NodeDocument extends Docume
/**
* Returns <code>true</code> if the given <code>revision</code> is marked
- * committed in <strong>this</strong> document.
+ * committed in <strong>this</strong> document including previous documents.
*
* @param revision the revision.
* @return <code>true</code> if committed; <code>false</code> otherwise.
*/
public boolean isCommitted(@Nonnull Revision revision) {
String rev = checkNotNull(revision).toString();
- String value = getRevisions().get(rev);
+ String value = getLocalRevisions().get(rev);
if (value != null) {
return Utils.isCommitted(value);
}
// check previous docs
- for (NodeDocument prev : getPreviousDocs(revision)) {
+ for (NodeDocument prev : getPreviousDocs(revision, REVISIONS)) {
if (prev.containsRevision(revision)) {
return prev.isCommitted(revision);
}
@@ -185,7 +186,7 @@ public class NodeDocument extends Docume
if (getLocalRevisions().containsKey(rev)) {
return true;
}
- for (NodeDocument prev : getPreviousDocs(revision)) {
+ for (NodeDocument prev : getPreviousDocs(revision, REVISIONS)) {
if (prev.containsRevision(revision)) {
return true;
}
@@ -240,11 +241,12 @@ public class NodeDocument extends Docume
*/
@CheckForNull
public String getCommitRootPath(String revision) {
- Map<String, Integer> valueMap = getCommitRoot();
- Integer depth = valueMap.get(revision);
+ Map<String, String> valueMap = getCommitRoot();
+ String depth = valueMap.get(revision);
if (depth != null) {
String p = Utils.getPathFromId(getId());
- return PathUtils.getAncestorPath(p, PathUtils.getDepth(p) - depth);
+ return PathUtils.getAncestorPath(p,
+ PathUtils.getDepth(p) - Integer.parseInt(depth));
} else {
return null;
}
@@ -581,6 +583,10 @@ public class NodeDocument extends Docume
*/
@Nonnull
public Iterable<UpdateOp> split(@Nonnull RevisionContext context) {
+ // only consider if there are enough commits
+ if (getLocalRevisions().size() + getLocalCommitRoot().size() <= REVISIONS_SPLIT_OFF_SIZE) {
+ return Collections.emptyList();
+ }
String id = getId();
SortedMap<Revision, Range> previous = getPreviousRanges();
// what's the most recent previous revision?
@@ -594,47 +600,68 @@ public class NodeDocument extends Docume
recentPrevious = rev;
}
}
- NavigableMap<Revision, String> splitRevs
- = new TreeMap<Revision, String>(context.getRevisionComparator());
- Map<String, String> revisions = getLocalRevisions();
- // only consider if there are enough revisions
- if (revisions.size() > REVISIONS_SPLIT_OFF_SIZE) {
- // collect commits of this cluster node after the
+ Map<String, NavigableMap<Revision, String>> splitValues
+ = new HashMap<String, NavigableMap<Revision, String>>();
+ for (String property : new String[]{REVISIONS, COMMIT_ROOT, DELETED}) {
+ NavigableMap<Revision, String> splitMap
+ = new TreeMap<Revision, String>(context.getRevisionComparator());
+ splitValues.put(property, splitMap);
+ Map<String, String> valueMap = getLocalMap(property);
+ // collect committed changes of this cluster node after the
// most recent previous split revision
- for (Map.Entry<String, String> entry : revisions.entrySet()) {
+ for (Map.Entry<String, String> entry : valueMap.entrySet()) {
Revision rev = Revision.fromString(entry.getKey());
if (rev.getClusterId() != context.getClusterId()) {
continue;
}
if (recentPrevious == null
|| isRevisionNewer(context, rev, recentPrevious)) {
- String commitValue = entry.getValue();
- if (Utils.isCommitted(commitValue)) {
- splitRevs.put(rev, commitValue);
+ if (isCommitted(rev)) {
+ splitMap.put(rev, entry.getValue());
}
}
}
}
+
List<UpdateOp> splitOps = Collections.emptyList();
- if (splitRevs.size() > REVISIONS_SPLIT_OFF_SIZE) {
+ int numValues = 0;
+ Revision high = null;
+ Revision low = null;
+ for (NavigableMap<Revision, String> splitMap : splitValues.values()) {
+ // keep the most recent in the main document
+ if (!splitMap.isEmpty()) {
+ splitMap.remove(splitMap.lastKey());
+ }
+ if (splitMap.isEmpty()) {
+ continue;
+ }
+ // remember highest / lowest revision
+ if (high == null || isRevisionNewer(context, splitMap.lastKey(), high)) {
+ high = splitMap.lastKey();
+ }
+ if (low == null || isRevisionNewer(context, low, splitMap.firstKey())) {
+ low = splitMap.firstKey();
+ }
+ numValues += splitMap.size();
+ }
+ if (high != null && low != null && numValues >= REVISIONS_SPLIT_OFF_SIZE) {
// enough revisions to split off
splitOps = new ArrayList<UpdateOp>(2);
- // keep the most recent in the main document
- splitRevs.remove(splitRevs.lastKey());
- // move the others to another document
- Revision high = splitRevs.lastEntry().getKey();
- Revision low = splitRevs.firstEntry().getKey();
+ // move to another document
UpdateOp main = new UpdateOp(id, false);
main.setMapEntry(PREVIOUS, high.toString(), low.toString());
UpdateOp old = new UpdateOp(Utils.getPreviousIdFor(id, high), true);
old.set(ID, old.getKey());
- for (Map.Entry<Revision, String> entry : splitRevs.entrySet()) {
- String r = entry.getKey().toString();
- main.removeMapEntry(REVISIONS, r);
- old.setMapEntry(REVISIONS, r, entry.getValue());
+ for (String property : splitValues.keySet()) {
+ NavigableMap<Revision, String> splitMap = splitValues.get(property);
+ for (Map.Entry<Revision, String> entry : splitMap.entrySet()) {
+ String r = entry.getKey().toString();
+ main.removeMapEntry(property, r);
+ old.setMapEntry(property, r, entry.getValue());
+ }
+ splitOps.add(old);
+ splitOps.add(main);
}
- splitOps.add(old);
- splitOps.add(main);
}
return splitOps;
}
@@ -688,20 +715,25 @@ public class NodeDocument extends Docume
}
/**
- * Returns previous {@link NodeDocument}, which include the given revision.
+ * Returns previous {@link NodeDocument}, which include entries for the
+ * property in the given revision.
* If the <code>revision</code> is <code>null</code>, then all previous
* documents are returned.
*
* @param revision the revision to match or <code>null</code>.
+ * @param property the name of a property.
* @return previous documents.
*/
- Iterable<NodeDocument> getPreviousDocs(final @Nullable Revision revision) {
+ Iterable<NodeDocument> getPreviousDocs(final @Nullable Revision revision,
+ final @Nonnull String property) {
+ checkNotNull(property);
Iterable<NodeDocument> docs = Iterables.transform(
Iterables.filter(getPreviousRanges().entrySet(),
new Predicate<Map.Entry<Revision, Range>>() {
@Override
public boolean apply(Map.Entry<Revision, Range> input) {
- return revision == null || input.getValue().includes(revision);
+ return revision == null
+ || input.getValue().includes(revision);
}
}), new Function<Map.Entry<Revision, Range>, NodeDocument>() {
@Nullable
@@ -723,7 +755,8 @@ public class NodeDocument extends Docume
if (input == null) {
return false;
}
- return revision == null || input.containsRevision(revision.toString());
+ return revision == null
+ || input.getLocalMap(property).containsKey(revision.toString());
}
});
}
@@ -753,6 +786,16 @@ public class NodeDocument extends Docume
return getLocalMap(REVISIONS);
}
+ @Nonnull
+ Map<String, String> getLocalCommitRoot() {
+ return getLocalMap(COMMIT_ROOT);
+ }
+
+ @Nonnull
+ Map<String, String> getLocalDeleted() {
+ return getLocalMap(DELETED);
+ }
+
//-------------------------< UpdateOp modifiers >---------------------------
public static void setModified(@Nonnull UpdateOp op,
@@ -784,7 +827,7 @@ public class NodeDocument extends Docume
@Nonnull Revision revision,
int commitRootDepth) {
checkNotNull(op).setMapEntry(COMMIT_ROOT,
- checkNotNull(revision).toString(), commitRootDepth);
+ checkNotNull(revision).toString(), String.valueOf(commitRootDepth));
}
public static void setDeleted(@Nonnull UpdateOp op,
@@ -807,14 +850,7 @@ public class NodeDocument extends Docume
if (containsRevision(rev)) {
return this;
}
- // check commit root
- Map<String, Integer> commitRoot = getCommitRoot();
- String commitRootPath = null;
- Integer depth = commitRoot.get(rev.toString());
- if (depth != null) {
- String p = Utils.getPathFromId(getId());
- commitRootPath = PathUtils.getAncestorPath(p, PathUtils.getDepth(p) - depth);
- }
+ String commitRootPath = getCommitRootPath(rev.toString());
if (commitRootPath == null) {
// shouldn't happen, either node is commit root for a revision
// or has a reference to the commit root
@@ -898,7 +934,7 @@ public class NodeDocument extends Docume
String value = getLocalRevisions().get(r);
if (value == null) {
// check previous
- for (NodeDocument prev : getPreviousDocs(revision)) {
+ for (NodeDocument prev : getPreviousDocs(revision, REVISIONS)) {
value = prev.getLocalRevisions().get(r);
if (value != null) {
break;
@@ -946,9 +982,9 @@ public class NodeDocument extends Docume
*/
@CheckForNull
private static String getLatestValue(@Nonnull RevisionContext context,
- @Nonnull Map<String, String> valueMap,
- @Nullable Revision min,
- @Nonnull Revision max) {
+ @Nonnull Map<String, String> valueMap,
+ @Nullable Revision min,
+ @Nonnull Revision max) {
String value = null;
Revision latestRev = null;
for (String r : valueMap.keySet()) {
@@ -978,12 +1014,7 @@ public class NodeDocument extends Docume
}
@Nonnull
- private Map<String, Integer> getCommitRoot() {
- @SuppressWarnings("unchecked")
- Map<String, Integer> commitRoot = (Map<String, Integer>) get(COMMIT_ROOT);
- if (commitRoot == null) {
- commitRoot = Collections.emptyMap();
- }
- return commitRoot;
+ private Map<String, String> getCommitRoot() {
+ return ValueMap.create(this, COMMIT_ROOT);
}
}
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/mongomk/ValueMap.java Mon Sep 16 09:16:19 2013
@@ -46,7 +46,7 @@ class ValueMap {
@Nonnull
public Iterator<Map.Entry<String, String>> iterator() {
return Iterators.concat(map.entrySet().iterator(), Iterators.concat(new Iterator<Iterator<Map.Entry<String, String>>>() {
- private final Iterator<NodeDocument> previous = doc.getPreviousDocs(null).iterator();
+ private final Iterator<NodeDocument> previous = doc.getPreviousDocs(null, property).iterator();
@Override
public boolean hasNext() {
@@ -68,7 +68,7 @@ class ValueMap {
@Override
public int size() {
int size = map.size();
- for (NodeDocument prev : doc.getPreviousDocs(null)) {
+ for (NodeDocument prev : doc.getPreviousDocs(null, property)) {
size += prev.getLocalMap(property).size();
}
return size;
@@ -92,7 +92,7 @@ class ValueMap {
return value;
}
Revision r = Revision.fromString(key.toString());
- for (NodeDocument prev : doc.getPreviousDocs(r)) {
+ for (NodeDocument prev : doc.getPreviousDocs(r, property)) {
value = prev.getLocalMap(property).get(key);
if (value != null) {
return value;
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java?rev=1523568&r1=1523567&r2=1523568&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/mongomk/DocumentSplitTest.java Mon Sep 16 09:16:19 2013
@@ -26,6 +26,7 @@ import com.google.common.collect.Sets;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
/**
@@ -40,16 +41,11 @@ public class DocumentSplitTest extends B
NodeDocument doc = store.find(Collection.NODES, Utils.getIdFromPath("/"));
assertNotNull(doc);
revisions.addAll(doc.getLocalRevisions().keySet());
-
- // MongoMK initializes with a root node with a single revision
- int numRevs = 1;
-
revisions.add(mk.commit("/", "+\"foo\":{}+\"bar\":{}", null, null));
- numRevs++;
// create nodes
- while (numRevs++ <= NodeDocument.REVISIONS_SPLIT_OFF_SIZE) {
- revisions.add(mk.commit("/", "+\"foo/node-" + numRevs + "\":{}" +
- "+\"bar/node-" + numRevs + "\":{}", null, null));
+ while (revisions.size() <= NodeDocument.REVISIONS_SPLIT_OFF_SIZE) {
+ revisions.add(mk.commit("/", "+\"foo/node-" + revisions.size() + "\":{}" +
+ "+\"bar/node-" + revisions.size() + "\":{}", null, null));
}
mk.runBackgroundOperations();
String head = mk.getHeadRevision();
@@ -69,4 +65,42 @@ public class DocumentSplitTest extends B
mk.setAsyncDelay(0);
mk.backgroundWrite();
}
+
+ @Test
+ public void splitDeleted() throws Exception {
+ DocumentStore store = mk.getDocumentStore();
+ Set<String> revisions = Sets.newHashSet();
+ mk.commit("/", "+\"foo\":{}", null, null);
+ NodeDocument doc = store.find(Collection.NODES, Utils.getIdFromPath("/foo"));
+ assertNotNull(doc);
+ revisions.addAll(doc.getLocalRevisions().keySet());
+ boolean create = false;
+ while (revisions.size() <= NodeDocument.REVISIONS_SPLIT_OFF_SIZE) {
+ if (create) {
+ revisions.add(mk.commit("/", "+\"foo\":{}", null, null));
+ } else {
+ revisions.add(mk.commit("/", "-\"foo\"", null, null));
+ }
+ create = !create;
+ }
+ mk.runBackgroundOperations();
+ String head = mk.getHeadRevision();
+ doc = store.find(Collection.NODES, Utils.getIdFromPath("/foo"));
+ assertNotNull(doc);
+ Map<String, String> deleted = doc.getLocalDeleted();
+ // one remaining in the local deleted map
+ assertEquals(1, deleted.size());
+ for (String r : revisions) {
+ Revision rev = Revision.fromString(r);
+ assertTrue(doc.containsRevision(rev));
+ assertTrue(doc.isCommitted(rev));
+ }
+ Node node = doc.getNodeAtRevision(mk, Revision.fromString(head));
+ // check status of node
+ if (create) {
+ assertNull(node);
+ } else {
+ assertNotNull(node);
+ }
+ }
}