You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2013/03/19 10:03:31 UTC
svn commit: r1458181 - in /jackrabbit/oak/trunk/oak-mongomk/src:
main/java/org/apache/jackrabbit/mongomk/prototype/
test/java/org/apache/jackrabbit/mongomk/prototype/
Author: thomasm
Date: Tue Mar 19 09:03:31 2013
New Revision: 1458181
URL: http://svn.apache.org/r1458181
Log:
OAK-619 Lock-free MongoMK implementation (fixes child list caching)
Modified:
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java Tue Mar 19 09:03:31 2013
@@ -106,8 +106,14 @@ public class Commit {
addedNodes.add(n.path);
}
- boolean isEmpty() {
- return operations.isEmpty();
+ /**
+ * Apply the changes to the document store and the cache.
+ */
+ void apply() {
+ if (!operations.isEmpty()) {
+ applyToDocumentStore();
+ applyToCache();
+ }
}
/**
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java Tue Mar 19 09:03:31 2013
@@ -26,8 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nonnull;
@@ -263,37 +261,39 @@ public class MongoMK implements MicroKer
return x.compareRevisionTime(previous) > 0;
}
- public Node.Children readChildren(final String path, final String nodeId,
- final Revision rev, final int limit) {
- try {
- return nodeChildrenCache.get(nodeId, new Callable<Children>() {
- @Override
- public Children call() throws Exception {
- String from = PathUtils.concat(path, "a");
- from = Utils.getIdFromPath(from);
- from = from.substring(0, from.length() - 1);
- String to = PathUtils.concat(path, "z");
- to = Utils.getIdFromPath(to);
- to = to.substring(0, to.length() - 2) + "0";
- List<Map<String, Object>> list = store.query(DocumentStore.Collection.NODES, from, to, limit);
- Children c = new Children(path, nodeId, rev);
- for (Map<String, Object> e : list) {
- // filter out deleted children
- if (getLiveRevision(e, rev) == null) {
- continue;
- }
- // TODO put the whole node in the cache
- String id = e.get(UpdateOp.ID).toString();
- String p = Utils.getPathFromId(id);
- c.children.add(p);
- }
- return c;
- }
- });
- } catch (ExecutionException e) {
- throw new IllegalStateException("Error occurred while fetching node children for node "+nodeId,e);
+ public Children getChildren(String path, Revision rev, int limit) {
+ checkRevisionAge(rev, path);
+ String key = path + "@" + rev;
+ Children children = nodeChildrenCache.getIfPresent(key);
+ if (children == null) {
+ children = readChildren(path, rev, limit);
+ if (children != null) {
+ nodeChildrenCache.put(key, children);
+ }
+ }
+ return children;
+ }
+
+ Node.Children readChildren(String path, Revision rev, int limit) {
+ String from = PathUtils.concat(path, "a");
+ from = Utils.getIdFromPath(from);
+ from = from.substring(0, from.length() - 1);
+ String to = PathUtils.concat(path, "z");
+ to = Utils.getIdFromPath(to);
+ to = to.substring(0, to.length() - 2) + "0";
+ List<Map<String, Object>> list = store.query(DocumentStore.Collection.NODES, from, to, limit);
+ Children c = new Children(path, rev);
+ for (Map<String, Object> e : list) {
+ // filter out deleted children
+ if (getLiveRevision(e, rev) == null) {
+ continue;
+ }
+ // TODO put the whole node in the cache
+ String id = e.get(UpdateOp.ID).toString();
+ String p = Utils.getPathFromId(id);
+ c.children.add(p);
}
-
+ return c;
}
private Node readNode(String path, Revision rev) {
@@ -429,17 +429,18 @@ public class MongoMK implements MicroKer
Revision toRev = Revision.fromString(toRevisionId);
// TODO this does not work well for large child node lists
// use a MongoDB index instead
- Children fromChildren = readChildren(path, from.getId(), fromRev, Integer.MAX_VALUE);
- Children toChildren = readChildren(path, to.getId(), toRev, Integer.MAX_VALUE);
+ Children fromChildren = getChildren(path, fromRev, Integer.MAX_VALUE);
+ Children toChildren = getChildren(path, toRev, Integer.MAX_VALUE);
Set<String> childrenSet = new HashSet<String>(toChildren.children);
for (String n : fromChildren.children) {
if (!childrenSet.contains(n)) {
w.tag('-').key(n).object().endObject().newline();
} else {
- // TODO currently all children seem to diff,
- // which is not necessarily the case
- // (compare write counters)
- w.tag('^').key(n).object().endObject().newline();
+ Node n1 = getNode(n, fromRev);
+ Node n2 = getNode(n, toRev);
+ if (!n1.equals(n2)) {
+ w.tag('^').key(n).object().endObject().newline();
+ }
}
}
childrenSet = new HashSet<String>(fromChildren.children);
@@ -491,7 +492,7 @@ public class MongoMK implements MicroKer
json.object();
n.append(json, includeId);
// FIXME: must not read all children!
- Children c = readChildren(path, n.getId(), rev, Integer.MAX_VALUE);
+ Children c = getChildren(path, rev, Integer.MAX_VALUE);
for (long i = offset; i < c.children.size(); i++) {
if (maxChildNodes-- <= 0) {
break;
@@ -577,10 +578,11 @@ public class MongoMK implements MicroKer
}
if (revisionId.startsWith("b")) {
// just commit to head currently
- applyCommit(commit);
+ commit.apply();
// remember branch commit
branchCommits.put(rev.toString(), revisionId.substring(1));
+ headRevision = commit.getRevision();
return "b" + rev.toString();
// String jsonBranch = branchCommits.remove(revisionId);
@@ -589,7 +591,8 @@ public class MongoMK implements MicroKer
// branchCommits.put(branchRev, jsonBranch);
// return branchRev;
}
- applyCommit(commit);
+ commit.apply();
+ headRevision = commit.getRevision();
return rev.toString();
}
@@ -628,8 +631,7 @@ public class MongoMK implements MicroKer
if (move) {
markAsDeleted(sourcePath, commit, false);
}
- Node.Children c = readChildren(sourcePath, n.getId(),
- baseRev, Integer.MAX_VALUE);
+ Node.Children c = getChildren(sourcePath, baseRev, Integer.MAX_VALUE);
for (String srcChildPath : c.children) {
String childName = PathUtils.getName(srcChildPath);
String destChildPath = PathUtils.concat(targetPath, childName);
@@ -651,7 +653,7 @@ public class MongoMK implements MicroKer
nodeCache.invalidate(path + "@" + rev);
if (n != null) {
- Node.Children c = readChildren(path, n.getId(), rev,
+ Node.Children c = getChildren(path, rev,
Integer.MAX_VALUE);
for (String childPath : c.children) {
markAsDeleted(childPath, commit, true);
@@ -703,15 +705,6 @@ public class MongoMK implements MicroKer
}
return revisionId;
}
-
- private void applyCommit(Commit commit) {
- headRevision = commit.getRevision();
- if (commit.isEmpty()) {
- return;
- }
- commit.applyToDocumentStore();
- commit.applyToCache();
- }
public static void parseAddNode(Commit commit, JsopReader t, String path) {
Node n = new Node(path, commit.getRevision());
@@ -849,8 +842,8 @@ public class MongoMK implements MicroKer
long newWriteCount = oldWriteCount + writeCountInc;
Children c = nodeChildrenCache.getIfPresent(path + "@" + (newWriteCount - 1));
if (isNew || (!isDelete && c != null)) {
- String id = path + "@" + newWriteCount;
- Children c2 = new Children(path, id, rev);
+ String key = path + "@" + rev;
+ Children c2 = new Children(path, rev);
TreeSet<String> set = new TreeSet<String>();
if (c != null) {
set.addAll(c.children);
@@ -858,14 +851,7 @@ public class MongoMK implements MicroKer
set.removeAll(removed);
set.addAll(added);
c2.children.addAll(set);
- if (nodeChildrenCache.getIfPresent(id) != null) {
- // TODO should not happend,
- // probably a problem with add/delete/add
- MicroKernelException e = new MicroKernelException(
- "New child list already cached for id " + id);
- LOG.error("Error updating the cache", e);
- }
- nodeChildrenCache.put(id, c2);
+ nodeChildrenCache.put(key, c2);
}
}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Node.java Tue Mar 19 09:03:31 2013
@@ -95,20 +95,40 @@ public class Node {
}
}
+ public void setWriteCount(long writeCount) {
+ this.writeCount = writeCount;
+ }
+
+ public long getWriteCount() {
+ return writeCount;
+ }
+
+ public int hashCode() {
+ return (int) writeCount ^ properties.size() ^ rev.hashCode();
+ }
+
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ } else if (!(obj instanceof Node)) {
+ return false;
+ }
+ Node other = (Node) obj;
+ return writeCount == other.writeCount;
+ }
+
/**
* A list of children for a node.
*/
static class Children {
final String path;
- final String id;
final Revision rev;
final ArrayList<String> children = new ArrayList<String>();
- Children(String path, String id, Revision rev) {
+ Children(String path, Revision rev) {
this.path = path;
- this.id = id;
this.rev = rev;
}
@@ -118,8 +138,4 @@ public class Node {
}
- public void setWriteCount(long writeCount) {
- this.writeCount = writeCount;
- }
-
}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java Tue Mar 19 09:03:31 2013
@@ -24,11 +24,36 @@ import java.util.TreeMap;
*/
public class UpdateOp {
+ /**
+ * The node id, which contains the depth of the path
+ * (0 for root, 1 for children of the root), and then the path.
+ */
static final String ID = "_id";
+
+ /**
+ * The number of write operations to this node.
+ */
static final String WRITE_COUNT = "_writeCount";
+
+ /**
+ * The list of recent revisions against this node, where this node is the
+ * root of the commit.
+ */
static final String REVISIONS = "_revisions";
+
+ /**
+ * The number of previous documents (documents that contain old revisions of
+ * this node). This property is only set if multiple documents per node
+ * exist. This is the case when a node is updated very often in a short
+ * time, such that the document gets very big.
+ */
static final String PREVIOUS = "_prev";
+
+ /**
+ * Whether this node is
+ */
static final String DELETED = "_deleted";
+ static final String MODIFIED = "_modified";
final String path;
Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java?rev=1458181&r1=1458180&r2=1458181&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java Tue Mar 19 09:03:31 2013
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTru
import org.apache.jackrabbit.mongomk.prototype.DocumentStore.Collection;
import org.apache.jackrabbit.mongomk.prototype.Node.Children;
+import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.Lists;
@@ -89,21 +90,35 @@ public class SimpleTest {
}
@Test
+ @Ignore
public void diff() {
MongoMK mk = createMK();
+
String rev0 = mk.getHeadRevision();
- // TODO
-// String rev1 = mk.commit("/", "+\"test\":{\"name\": \"Hello\"}", null, null);
-// String rev2 = mk.commit("/", "-\"test\"", null, null);
-// String rev3 = mk.commit("/", "+\"test\":{\"name\": \"Hallo\"}", null, null);
-// String test0 = mk.getNodes("/test", rev0, 0, 0, Integer.MAX_VALUE, null);
-// assertNull(null, test0);
-// String test1 = mk.getNodes("/test", rev1, 0, 0, Integer.MAX_VALUE, null);
-// assertEquals("{\"name\":\"Hello\",\":childNodeCount\":0}", test1);
-// String test2 = mk.getNodes("/test", rev2, 0, 0, Integer.MAX_VALUE, null);
-// assertNull(null, test2);
-// String test3 = mk.getNodes("/test", rev3, 0, 0, Integer.MAX_VALUE, null);
-// assertEquals("{\"name\":\"Hallo\",\":childNodeCount\":0}", test3);
+ String rev1 = mk.commit("/", "+\"t1\":{}", null, null);
+ String rev2 = mk.commit("/", "+\"t2\":{}", null, null);
+ String rev3 = mk.commit("/", "+\"t3\":{}", null, null);
+ String rev4 = mk.commit("/", "^\"t2/x\":1", null, null);
+
+ String r0 = mk.getNodes("/", rev0, 0, 0, Integer.MAX_VALUE, null);
+ assertEquals("{\":childNodeCount\":0}", r0);
+ String r1 = mk.getNodes("/", rev1, 0, 0, Integer.MAX_VALUE, null);
+ assertEquals("{\"t1\":{},\":childNodeCount\":1}", r1);
+ String r2 = mk.getNodes("/", rev2, 0, 0, Integer.MAX_VALUE, null);
+ assertEquals("{\"t1\":{},\"t2\":{},\":childNodeCount\":2}", r2);
+ String r3 = mk.getNodes("/", rev3, 0, 0, Integer.MAX_VALUE, null);
+ assertEquals("{\"t1\":{},\"t2\":{},\"t3\":{},\":childNodeCount\":3}", r3);
+
+ String diff01 = mk.diff(rev0, rev1, "/", 0).trim();
+ assertEquals("+\"/t1\":{}", diff01);
+ String diff12 = mk.diff(rev1, rev2, "/", 0).trim();
+ assertEquals("+\"/t2\":{}", diff12);
+ String diff23 = mk.diff(rev2, rev3, "/", 0).trim();
+ assertEquals("+\"/t3\":{}", diff23);
+ String diff13 = mk.diff(rev1, rev3, "/", 0).trim();
+ assertEquals("+\"/t2\":{}\n+\"/t3\":{}", diff13);
+ String diff34 = mk.diff(rev3, rev4, "/", 0).trim();
+ assertEquals("^\"/t2\":{}", diff34);
mk.dispose();
}
@@ -178,15 +193,13 @@ public class SimpleTest {
String test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);
assertEquals("{\"name\":\"Hello\",\":childNodeCount\":0}", test);
- rev = mk.commit("/test", "+\"a\":{\"name\": \"World\"}", null, null);
- rev = mk.commit("/test", "+\"b\":{\"name\": \"!\"}", null, null);
- test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);
+ String r0 = mk.commit("/test", "+\"a\":{\"name\": \"World\"}", null, null);
+ String r1 = mk.commit("/test", "+\"b\":{\"name\": \"!\"}", null, null);
+ test = mk.getNodes("/test", r0, 0, 0, Integer.MAX_VALUE, null);
Children c;
- c = mk.readChildren("/", "1",
- Revision.fromString(rev), Integer.MAX_VALUE);
+ c = mk.getChildren("/", Revision.fromString(r0), Integer.MAX_VALUE);
assertEquals("/: [/test]", c.toString());
- c = mk.readChildren("/test", "2",
- Revision.fromString(rev), Integer.MAX_VALUE);
+ c = mk.getChildren("/test", Revision.fromString(r1), Integer.MAX_VALUE);
assertEquals("/test: [/test/a, /test/b]", c.toString());
rev = mk.commit("", "^\"/test\":1", null, null);
@@ -252,23 +265,22 @@ public class SimpleTest {
public void testDeletion() {
MongoMK mk = createMK();
- String rev = mk.commit("/", "+\"testDel\":{\"name\": \"Hello\"}", null,
- null);
- rev = mk.commit("/testDel", "+\"a\":{\"name\": \"World\"}", null, null);
- rev = mk.commit("/testDel", "+\"b\":{\"name\": \"!\"}", null, null);
- rev = mk.commit("/testDel", "+\"c\":{\"name\": \"!\"}", null, null);
+ mk.commit("/", "+\"testDel\":{\"name\": \"Hello\"}", null, null);
+ mk.commit("/testDel", "+\"a\":{\"name\": \"World\"}", null, null);
+ mk.commit("/testDel", "+\"b\":{\"name\": \"!\"}", null, null);
+ String r1 = mk.commit("/testDel", "+\"c\":{\"name\": \"!\"}", null, null);
- Children c = mk.readChildren("/testDel", "1", Revision.fromString(rev),
+ Children c = mk.getChildren("/testDel", Revision.fromString(r1),
Integer.MAX_VALUE);
assertEquals(3, c.children.size());
- rev = mk.commit("/testDel", "-\"c\"", null, null);
- c = mk.readChildren("/testDel", "2", Revision.fromString(rev),
+ String r2 = mk.commit("/testDel", "-\"c\"", null, null);
+ c = mk.getChildren("/testDel", Revision.fromString(r2),
Integer.MAX_VALUE);
assertEquals(2, c.children.size());
- rev = mk.commit("/", "-\"testDel\"", null, null);
- Node n = mk.getNode("/testDel", Revision.fromString(rev));
+ String r3 = mk.commit("/", "-\"testDel\"", null, null);
+ Node n = mk.getNode("/testDel", Revision.fromString(r3));
assertNull(n);
}