You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2013/03/01 14:19:46 UTC
svn commit: r1451586 - in /jackrabbit/oak/trunk/oak-mongomk/src:
main/java/org/apache/jackrabbit/mongomk/prototype/
test/java/org/apache/jackrabbit/mongomk/prototype/
Author: thomasm
Date: Fri Mar 1 13:19:46 2013
New Revision: 1451586
URL: http://svn.apache.org/r1451586
Log:
OAK-619 Lock-free MongoMK implementation (bugfixes)
Modified:
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Commit.java Fri Mar 1 13:19:46 2013
@@ -27,12 +27,16 @@ import org.apache.jackrabbit.mk.json.Jso
import org.apache.jackrabbit.mk.json.JsopWriter;
import org.apache.jackrabbit.mongomk.prototype.DocumentStore.Collection;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A higher level object representing a commit.
*/
public class Commit {
-
+
+ private static final Logger LOG = LoggerFactory.getLogger(Commit.class);
+
/**
* The maximum size of a document. If it is larger, it is split.
*/
@@ -93,7 +97,9 @@ public class Commit {
void addNode(Node n) {
if (operations.containsKey(n.path)) {
- throw new MicroKernelException("Node already added: " + n.path);
+ String msg = "Node already added: " + n.path;
+ LOG.error(msg);
+ throw new MicroKernelException(msg);
}
operations.put(n.path, n.asOperation(true));
addedNodes.add(n.path);
@@ -145,7 +151,13 @@ public class Commit {
}
try {
if (newNodes.size() > 0) {
- store.create(Collection.NODES, newNodes);
+ if (!store.create(Collection.NODES, newNodes)) {
+ for (UpdateOp op : newNodes) {
+ op.unset(UpdateOp.ID);
+ op.addMapEntry(UpdateOp.DELETED + "." + revision.toString(), "false");
+ createOrUpdateNode(store, op);
+ }
+ }
}
for (UpdateOp op : changedNodes) {
createOrUpdateNode(store, op);
@@ -158,7 +170,9 @@ public class Commit {
operations.put(commitRoot, root);
}
} catch (MicroKernelException e) {
- throw new MicroKernelException("Exception committing " + diff.toString(), e);
+ String msg = "Exception committing " + diff.toString();
+ LOG.error(msg, e);
+ throw new MicroKernelException(msg, e);
}
}
@@ -193,8 +207,8 @@ public class Commit {
}
private UpdateOp[] splitDocument(Map<String, Object> map) {
- String path = (String) map.get(UpdateOp.PATH);
String id = (String) map.get(UpdateOp.ID);
+ String path = id.substring(1);
Long previous = (Long) map.get(UpdateOp.PREVIOUS);
if (previous == null) {
previous = 0L;
@@ -206,9 +220,7 @@ public class Commit {
main.set(UpdateOp.PREVIOUS, previous);
for (Entry<String, Object> e : map.entrySet()) {
String key = e.getKey();
- if (key.equals(UpdateOp.PATH)) {
- // ok
- } else if (key.equals(UpdateOp.ID)) {
+ if (key.equals(UpdateOp.ID)) {
// ok
} else if (key.equals(UpdateOp.PREVIOUS)) {
// ok
@@ -278,6 +290,7 @@ public class Commit {
UpdateOp op = operations.get(path);
boolean isNew = op != null && op.isNew;
boolean isWritten = op != null;
+ boolean isDelete = op != null && op.isDelete;
long writeCountInc = mk.getWriteCountIncrement(path);
Long writeCount = writeCounts.get(path);
if (writeCount == null) {
@@ -293,7 +306,7 @@ public class Commit {
}
}
mk.applyChanges(revision, path,
- isNew, isWritten,
+ isNew, isDelete, isWritten,
writeCount, writeCountInc,
added, removed);
}
@@ -328,6 +341,7 @@ public class Commit {
public void removeNode(String path) {
removedNodes.add(path);
UpdateOp op = getUpdateOperationForNode(path);
+ op.setDelete(true);
op.addMapEntry(UpdateOp.DELETED + "." + revision.toString(), "true");
long increment = mk.getWriteCountIncrement(path);
op.increment(UpdateOp.WRITE_COUNT, 1 + increment);
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/DocumentStore.java Fri Mar 1 13:19:46 2013
@@ -49,7 +49,14 @@ public interface DocumentStore {
*/
void remove(Collection collection, String key);
- void create(Collection collection, List<UpdateOp> updateOps);
+ /**
+ * Try to create a list of documents.
+ *
+ * @param collection the collection
+ * @param updateOps the list of documents to add
+ * @return true if this worked (if none of the documents already existed)
+ */
+ boolean create(Collection collection, List<UpdateOp> updateOps);
/**
* Create or update a document. For MongoDb, this is using "findAndModify" with
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MemoryDocumentStore.java Fri Mar 1 13:19:46 2013
@@ -177,10 +177,17 @@ public class MemoryDocumentStore impleme
}
@Override
- public void create(Collection collection, List<UpdateOp> updateOps) {
+ public boolean create(Collection collection, List<UpdateOp> updateOps) {
+ ConcurrentSkipListMap<String, Map<String, Object>> map = getMap(collection);
+ for (UpdateOp op : updateOps) {
+ if (map.containsKey(op.key)) {
+ return false;
+ }
+ }
for (UpdateOp op : updateOps) {
createOrUpdate(collection, op);
}
+ return true;
}
public String toString() {
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStore.java Fri Mar 1 13:19:46 2013
@@ -25,6 +25,8 @@ import java.util.Map.Entry;
import org.apache.jackrabbit.mk.api.MicroKernelException;
import org.apache.jackrabbit.mongomk.prototype.MongoMK.Cache;
import org.apache.jackrabbit.mongomk.prototype.UpdateOp.Operation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
@@ -40,9 +42,10 @@ import com.mongodb.WriteResult;
* A document store that uses MongoDB as the backend.
*/
public class MongoDocumentStore implements DocumentStore {
-
- private static final boolean LOG = false;
- private static final boolean LOG_TIME = true;
+
+ private static final Logger LOG = LoggerFactory.getLogger(MongoDocumentStore.class);
+
+ private static final boolean LOG_TIME = false;
private final DBCollection nodesCollection;
@@ -216,7 +219,7 @@ public class MongoDocumentStore implemen
}
@Override
- public void create(Collection collection, List<UpdateOp> updateOps) {
+ public boolean create(Collection collection, List<UpdateOp> updateOps) {
log("create", updateOps);
ArrayList<Map<String, Object>> maps = new ArrayList<Map<String, Object>>();
DBObject[] inserts = new DBObject[updateOps.size()];
@@ -259,7 +262,7 @@ public class MongoDocumentStore implemen
try {
WriteResult writeResult = dbCollection.insert(inserts, WriteConcern.SAFE);
if (writeResult.getError() != null) {
- throw new MicroKernelException("Batch create failed: " + writeResult.getError());
+ return false;
}
synchronized (cache) {
for (Map<String, Object> map : maps) {
@@ -269,8 +272,9 @@ public class MongoDocumentStore implemen
}
}
}
+ return true;
} catch (MongoException e) {
- throw new MicroKernelException("Batch create failed", e);
+ return false;
}
} finally {
end(start);
@@ -319,19 +323,19 @@ public class MongoDocumentStore implemen
@Override
public void dispose() {
- if (LOG_TIME) {
- System.out.println("MongoDB time: " + time);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("MongoDB time: " + time);
}
nodesCollection.getDB().getMongo().close();
}
- private static void log(Object... args) {
- if (LOG) {
- String msg = Arrays.toString(args);
- if (msg.length() > 10000) {
- msg = msg.length() + ": " + msg;
+ private static void log(String message, Object... args) {
+ if (LOG.isDebugEnabled()) {
+ String argList = Arrays.toString(args);
+ if (argList.length() > 10000) {
+ argList = argList.length() + ": " + argList;
}
- System.out.println(msg);
+ LOG.debug(message + argList);
}
}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java Fri Mar 1 13:19:46 2013
@@ -268,21 +268,28 @@ public class MongoMK implements MicroKer
@SuppressWarnings("unchecked")
Map<String, String> valueMap = (Map<String, String>) v;
if (valueMap != null) {
- Revision latestRev = null;
- for (String r : valueMap.keySet()) {
- Revision propRev = Revision.fromString(r);
- if (includeRevision(propRev, rev)) {
- if (latestRev == null || isRevisionNewer(propRev, latestRev)) {
- latestRev = propRev;
- n.setProperty(key, valueMap.get(r));
- }
- }
- }
+ String value = getLatestValue(valueMap, rev);
+ n.setProperty(key, value);
}
}
n.setWriteCount(writeCount);
return n;
}
+
+ private String getLatestValue(Map<String, String> valueMap, Revision rev) {
+ String value = null;
+ Revision latestRev = null;
+ for (String r : valueMap.keySet()) {
+ Revision propRev = Revision.fromString(r);
+ if (includeRevision(propRev, rev)) {
+ if (latestRev == null || isRevisionNewer(propRev, latestRev)) {
+ latestRev = propRev;
+ value = valueMap.get(r);
+ }
+ }
+ }
+ return value;
+ }
@Override
public String getHeadRevision() throws MicroKernelException {
@@ -510,22 +517,15 @@ public class MongoMK implements MicroKer
Map<String, String> valueMap = (Map<String, String>) nodeProps
.get(UpdateOp.DELETED);
if (valueMap != null) {
- for (Map.Entry<String, String> e : valueMap.entrySet()) {
- // TODO What if multiple revisions are there?. Should we sort
- // them and then
- // determine include revision based on that
- Revision propRev = Revision.fromString(e.getKey());
- if (includeRevision(propRev, rev)) {
- if ("true".equals(e.getValue())) {
- return true;
- }
- }
+ String value = getLatestValue(valueMap, rev);
+ if ("true".equals(value)) {
+ return true;
}
}
return false;
}
- private static String stripBranchRevMarker(String revisionId){
+ private static String stripBranchRevMarker(String revisionId) {
if (revisionId.startsWith("b")) {
return revisionId.substring(1);
}
@@ -679,7 +679,7 @@ public class MongoMK implements MicroKer
}
public void applyChanges(Revision rev, String path,
- boolean isNew, boolean isWritten,
+ boolean isNew, boolean isDelete, boolean isWritten,
long oldWriteCount, long writeCountInc,
ArrayList<String> added, ArrayList<String> removed) {
if (!isWritten) {
@@ -693,7 +693,7 @@ public class MongoMK implements MicroKer
}
long newWriteCount = oldWriteCount + writeCountInc;
Children c = nodeChildrenCache.get(path + "@" + (newWriteCount - 1));
- if (isNew || c != null) {
+ if (isNew || (!isDelete && c != null)) {
String id = path + "@" + newWriteCount;
Children c2 = new Children(path, id, rev);
TreeSet<String> set = new TreeSet<String>();
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/UpdateOp.java Fri Mar 1 13:19:46 2013
@@ -25,7 +25,6 @@ import java.util.TreeMap;
public class UpdateOp {
static final String ID = "_id";
- static final String PATH = "_path";
static final String WRITE_COUNT = "_writeCount";
static final String REVISIONS = "_revisions";
static final String PREVIOUS = "_prev";
@@ -36,6 +35,7 @@ public class UpdateOp {
final String key;
final boolean isNew;
+ boolean isDelete;
final Map<String, Operation> changes = new TreeMap<String, Operation>();
@@ -46,6 +46,7 @@ public class UpdateOp {
* @param path the node path (for nodes)
* @param key the primary key
* @param isNew whether this is a new document
+ * @param isDelete whether the _deleted property is set
* @param rev the revision
*/
UpdateOp(String path, String key, boolean isNew) {
@@ -62,6 +63,10 @@ public class UpdateOp {
return isNew;
}
+ void setDelete(boolean isDelete) {
+ this.isDelete = isDelete;
+ }
+
/**
* Add a new map entry for this revision.
*
@@ -100,6 +105,15 @@ public class UpdateOp {
op.value = value;
changes.put(property, op);
}
+
+ /**
+ * Do not set the property (after it has been set).
+ *
+ * @param property the property name
+ */
+ void unset(String property) {
+ changes.remove(property);
+ }
/**
* Increment the value.
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/Utils.java Fri Mar 1 13:19:46 2013
@@ -20,6 +20,8 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;
+import org.bson.types.ObjectId;
+
/**
* Utility methods.
*/
@@ -49,5 +51,15 @@ public class Utils {
}
return size;
}
+
+ /**
+ * Generate a unique cluster id, similar to the machine id field in MongoDB ObjectId objects.
+ *
+ * @return the unique machine id
+ */
+ public static int getUniqueClusterId() {
+ ObjectId objId = new ObjectId();
+ return objId._machine();
+ }
}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java?rev=1451586&r1=1451585&r2=1451586&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/SimpleTest.java Fri Mar 1 13:19:46 2013
@@ -58,17 +58,28 @@ public class SimpleTest {
public void addNodeGetNode() {
MongoMK mk = new MongoMK();
Revision rev = mk.newRevision();
- Node n = new Node("/", rev);
+ Node n = new Node("/test", rev);
n.setProperty("name", "Hello");
UpdateOp op = n.asOperation(true);
DocumentStore s = mk.getDocumentStore();
- s.create(Collection.NODES, Lists.newArrayList(op));
- Node n2 = mk.getNode("/", rev);
+ assertTrue(s.create(Collection.NODES, Lists.newArrayList(op)));
+ Node n2 = mk.getNode("/test", rev);
assertEquals("Hello", n2.getProperty("name"));
mk.dispose();
}
@Test
+ public void reAddDeleted() {
+ MongoMK mk = createMK();
+ String rev = mk.commit("/", "+\"test\":{\"name\": \"Hello\"}", null, null);
+ rev = mk.commit("/", "-\"test\"", rev, null);
+ rev = mk.commit("/", "+\"test\":{\"name\": \"Hallo\"}", null, null);
+ String test = mk.getNodes("/test", rev, 0, 0, Integer.MAX_VALUE, null);
+ assertEquals("{\"name\":\"Hallo\",\":childNodeCount\":0}", test);
+ mk.dispose();
+ }
+
+ @Test
public void commit() {
MongoMK mk = createMK();