You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by th...@apache.org on 2013/02/18 13:59:42 UTC
svn commit: r1447234 - in /jackrabbit/oak/trunk/oak-mongomk/src:
main/java/org/apache/jackrabbit/mongomk/prototype/
test/java/org/apache/jackrabbit/mongomk/prototype/
Author: thomasm
Date: Mon Feb 18 12:59:42 2013
New Revision: 1447234
URL: http://svn.apache.org/r1447234
Log:
OAK-619 Lock-free MongoMK implementation (sharding setup, test, docs)
Added:
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt
Modified:
jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java
Modified: jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java?rev=1447234&r1=1447233&r2=1447234&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/main/java/org/apache/jackrabbit/mongomk/prototype/MongoMK.java Mon Feb 18 12:59:42 2013
@@ -30,8 +30,11 @@ import org.apache.jackrabbit.mk.blobs.Me
import org.apache.jackrabbit.mk.json.JsopReader;
import org.apache.jackrabbit.mk.json.JsopStream;
import org.apache.jackrabbit.mk.json.JsopTokenizer;
+import org.apache.jackrabbit.mongomk.impl.blob.MongoBlobStore;
import org.apache.jackrabbit.oak.commons.PathUtils;
+import com.mongodb.DB;
+
/**
* A MicroKernel implementation that stores the data in a MongoDB.
*/
@@ -92,11 +95,23 @@ public class MongoMK implements MicroKer
private Thread backgroundThread;
/**
- * Create a new in-memory MongoMK to be used for testing.
+ * Create a new in-memory MongoMK used for testing.
*/
public MongoMK() {
this(new MemoryDocumentStore(), new MemoryBlobStore(), 0);
}
+
+ /**
+ * Create a new MongoMK.
+ *
+ * @param db the MongoDB connection (null for in-memory)
+ * @param clusterId the cluster id (must be unique)
+ */
+ public MongoMK(DB db, int clusterId) {
+ this(db == null ? new MemoryDocumentStore() : new MongoDocumentStore(db),
+ db == null ? new MemoryBlobStore() : new MongoBlobStore(db),
+ clusterId);
+ }
/**
* Create a new MongoMK.
Added: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java?rev=1447234&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java (added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/BlobTest.java Mon Feb 18 12:59:42 2013
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mongomk.prototype;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.junit.Test;
+
+import com.mongodb.DB;
+
+/**
+ * Tests the blob store.
+ */
+public class BlobTest {
+
+ // private static boolean MONGO_DB = true;
+ // private static long TOTAL_SIZE = 1 * 1024 * 1024 * 1024;
+ // private static int DOCUMENT_COUNT = 10;
+
+ private static boolean MONGO_DB = false;
+ private static long TOTAL_SIZE = 1 * 1024 * 1024;
+ private static int DOCUMENT_COUNT = 10;
+
+ DB openMongoConnection() {
+ return MONGO_DB ? MongoUtils.getConnection().getDB() : null;
+ }
+
+ void dropCollections() {
+ if (MONGO_DB) {
+ MongoUtils.dropCollections(MongoUtils.getConnection().getDB());
+ }
+ }
+
+ @Test
+ public void addBlobs() throws Exception {
+ MongoMK mk = new MongoMK(openMongoConnection(), 0);
+ long blobSize = TOTAL_SIZE / DOCUMENT_COUNT;
+ ArrayList<String> blobIds = new ArrayList<String>();
+ // use a new seed each time, to allow running the test multiple times
+ Random r = new Random();
+ for (int i = 0; i < DOCUMENT_COUNT; i++) {
+ log("writing " + i + "/" + DOCUMENT_COUNT);
+ String id = mk.write(new RandomStream(blobSize, r.nextInt()));
+ blobIds.add(id);
+ }
+ for (String id : blobIds) {
+ assertEquals(blobSize, mk.getLength(id));
+ }
+ mk.dispose();
+ }
+
+ private static void log(String s) {
+ if (TOTAL_SIZE < 10 * 1024 * 1024) {
+ return;
+ }
+ System.out.println(s);
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java?rev=1447234&r1=1447233&r2=1447234&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java (original)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/MongoDocumentStoreTest.java Mon Feb 18 12:59:42 2013
@@ -103,7 +103,6 @@ public class MongoDocumentStoreTest {
}
@Test
- @Ignore
public void addLotsOfNodes() throws Exception {
char[] nPrefix = new char[]{'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'};
Added: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java?rev=1447234&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java (added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/RandomStream.java Mon Feb 18 12:59:42 2013
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.mongomk.prototype;
+
+import java.io.InputStream;
+import java.util.Random;
+
+/**
+ * A pseudo-random stream.
+ */
+public class RandomStream extends InputStream {
+
+ private long pos, size;
+ private Random random;
+
+ RandomStream(long size, int seed) {
+ this.size = size;
+ this.random = new Random(seed);
+ }
+
+ public int read() {
+ byte[] data = new byte[1];
+ int len = read(data, 0, 1);
+ return len <= 0 ? len : data[0] & 255;
+ }
+
+ public int read(byte[] b, int off, int len) {
+ if (pos >= size) {
+ return -1;
+ }
+ len = (int) Math.min(size - pos, len);
+ long end = off + len;
+ while (off < end) {
+ int r = random.nextInt();
+ b[off++] = (byte) (r ^ (r >>> 16));
+ }
+ pos += len;
+ return len;
+ }
+
+}
Added: jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt?rev=1447234&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt (added)
+++ jackrabbit/oak/trunk/oak-mongomk/src/test/java/org/apache/jackrabbit/mongomk/prototype/setup.txt Mon Feb 18 12:59:42 2013
@@ -0,0 +1,62 @@
+== MongoDB cluster setup ==
+
+This script will create a MongoDB cluster with 3 shards:
+
+=== Stop
+
+./mongo --port=27017
+use admin
+db.shutdownServer()
+exit
+./mongo --port=10001
+use admin
+db.shutdownServer()
+exit
+./mongo --port=10002
+use admin
+db.shutdownServer()
+exit
+./mongo --port=10003
+use admin
+db.shutdownServer()
+exit
+./mongo --port=20001
+use admin
+db.shutdownServer()
+exit
+ps -ef | grep mongo
+
+=== Clean, Init, and Start
+
+rm -rf db
+mkdir db
+mkdir db/shard1
+mkdir db/shard2
+mkdir db/shard3
+mkdir db/config
+./mongod --shardsvr --port=10001 --dbpath db/shard1 --fork --logpath db/shard1.log
+./mongod --shardsvr --port=10002 --dbpath db/shard2 --fork --logpath db/shard2.log
+./mongod --shardsvr --port=10003 --dbpath db/shard3 --fork --logpath db/shard3.log
+./mongod --configsvr --port 20001 --dbpath=db/config --fork --logpath db/config.log
+./mongos --port 27017 --configdb localhost:20001 --fork --logpath db/mongos.log
+./mongo --host localhost --port 27017
+sh.addShard("localhost:10001")
+sh.addShard("localhost:10002")
+sh.addShard("localhost:10003")
+use MongoMKDB
+db.createCollection("nodes")
+db.createCollection("blobs")
+sh.enableSharding("MongoMKDB")
+sh.shardCollection("MongoMKDB.nodes", { "_id": 1 }, true)
+sh.shardCollection("MongoMKDB.blobs", { "_id": 1 }, true)
+exit
+
+=== Other
+
+Display sharding status:
+
+du -h .
+./mongo --host localhost --port 27017
+db.printShardingStatus()
+exit
+