You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@jackrabbit.apache.org by st...@apache.org on 2012/03/11 21:01:51 UTC
svn commit: r1299437 - in /jackrabbit/sandbox/jackrabbit-mongo-persistence:
./ src/main/java/org/apache/jackrabbit/core/data/mongo/
src/main/java/org/apache/jackrabbit/core/persistence/mongo/
Author: stefan
Date: Sun Mar 11 20:01:51 2012
New Revision: 1299437
URL: http://svn.apache.org/viewvc?rev=1299437&view=rev
Log:
experimental mongo-based pm & ds (wip)
Added:
jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataRecord.java
jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataStore.java
- copied, changed from r1299244, jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/data/mongo/MongoDataStore.java
jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/Utils.java
Removed:
jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/data/mongo/MongoDataStore.java
Modified:
jackrabbit/sandbox/jackrabbit-mongo-persistence/README.txt
jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoPersistenceManager.java
Modified: jackrabbit/sandbox/jackrabbit-mongo-persistence/README.txt
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-mongo-persistence/README.txt?rev=1299437&r1=1299436&r2=1299437&view=diff
==============================================================================
--- jackrabbit/sandbox/jackrabbit-mongo-persistence/README.txt (original)
+++ jackrabbit/sandbox/jackrabbit-mongo-persistence/README.txt Sun Mar 11 20:01:51 2012
@@ -8,7 +8,7 @@ Both MongoPersistenceManager and MongoDa
- host -> defaults to "127.0.0.1"
- port -> defaults to "27017"
-- dbName -> defaults to "rep"
+- dbName -> defaults to "rep" (MongoPersistenceManager) and "datastore" (MongoDataStore)
- TODO document all configuration parameters
MongoPersistenceManager also supports the following parameters:
Added: jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataRecord.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataRecord.java?rev=1299437&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataRecord.java (added)
+++ jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataRecord.java Sun Mar 11 20:01:51 2012
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.persistence.mongo;
+
+import com.mongodb.gridfs.GridFSDBFile;
+import org.apache.jackrabbit.core.data.AbstractDataRecord;
+import org.apache.jackrabbit.core.data.DataIdentifier;
+import org.apache.jackrabbit.core.data.DataStoreException;
+
+import java.io.BufferedInputStream;
+import java.io.InputStream;
+import java.util.Date;
+
+/**
+ * Data record that is stored in a mongodb
+ */
+public class MongoDataRecord extends AbstractDataRecord {
+
+ protected final GridFSDBFile resource;
+
+ /**
+ * Creates a data record based on the given identifier and length.
+ *
+ * @param identifier data identifier
+ * @param resource remote resource
+ */
+ public MongoDataRecord(DataIdentifier identifier, GridFSDBFile resource) {
+ super(identifier);
+ this.resource = resource;
+ }
+
+ @Override
+ public long getLength() throws DataStoreException {
+ // TODO touch remote resource?
+/*
+ resource.put("uploadDate", new Date());
+ resource.save();
+*/
+ return resource.getLength();
+ }
+
+ @Override
+ public InputStream getStream() throws DataStoreException {
+ // TODO touch remote resource?
+/*
+ resource.put("uploadDate", new Date());
+ resource.save();
+*/
+ return new BufferedInputStream(resource.getInputStream());
+ }
+
+ @Override
+ public long getLastModified() {
+ return resource.getUploadDate().getTime();
+ }
+}
Copied: jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataStore.java (from r1299244, jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/data/mongo/MongoDataStore.java)
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataStore.java?p2=jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataStore.java&p1=jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/data/mongo/MongoDataStore.java&r1=1299244&r2=1299437&rev=1299437&view=diff
==============================================================================
--- jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/data/mongo/MongoDataStore.java (original)
+++ jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoDataStore.java Sun Mar 11 20:01:51 2012
@@ -14,28 +14,34 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.jackrabbit.core.data.mongo;
+package org.apache.jackrabbit.core.persistence.mongo;
import com.mongodb.BasicDBObject;
import com.mongodb.DB;
-import com.mongodb.DBCollection;
+import com.mongodb.DBCursor;
import com.mongodb.Mongo;
import com.mongodb.ServerAddress;
import com.mongodb.WriteConcern;
import com.mongodb.gridfs.GridFS;
+import com.mongodb.gridfs.GridFSDBFile;
+import com.mongodb.gridfs.GridFSFile;
+import com.mongodb.gridfs.GridFSInputFile;
import org.apache.jackrabbit.core.data.DataIdentifier;
import org.apache.jackrabbit.core.data.DataRecord;
import org.apache.jackrabbit.core.data.DataStore;
import org.apache.jackrabbit.core.data.DataStoreException;
-import org.apache.jackrabbit.core.persistence.mongo.MongoConfig;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.jcr.RepositoryException;
+import java.io.IOException;
import java.io.InputStream;
import java.net.UnknownHostException;
+import java.util.ArrayList;
import java.util.Iterator;
+import static org.apache.jackrabbit.core.persistence.mongo.Utils.encodeName;
+
/**
*
*/
@@ -52,8 +58,33 @@ public class MongoDataStore implements D
protected DB db;
protected GridFS fs;
+ protected String dbName = "datastore";
+
+ /**
+ * The minimum size of an object that should be stored in this data store.
+ */
+ protected int minRecordLength = 1024;
+
+ /**
+ * The minimum modified date. If a file is accessed (read or write) with a modified date
+ * older than this value, the modified date is updated to the current time.
+ */
+ protected long minModifiedDate;
+
//--------------------------------------< configuration getters & setters >
+ public String getDbName() {
+ return dbName;
+ }
+
+ public void setDbName(String dbName) {
+ this.dbName = dbName;
+ }
+
+ public void setMinRecordLength(int minRecordLength) {
+ this.minRecordLength = minRecordLength;
+ }
+
public String getHost() {
return conf.getHost();
}
@@ -192,10 +223,13 @@ public class MongoDataStore implements D
throw new RepositoryException("failed to initialize MongoDataStore", e);
}
- db = con.getDB("datastore");
+ db = con.getDB(encodeName(dbName));
db.setWriteConcern(WriteConcern.SAFE);
fs = new GridFS(db);
+ db.getCollection("fs.files").ensureIndex(
+ new BasicDBObject("md5", 1),
+ new BasicDBObject("unique", true));
initialized = true;
}
@@ -214,34 +248,103 @@ public class MongoDataStore implements D
}
public DataRecord getRecordIfStored(DataIdentifier identifier) throws DataStoreException {
- return null; // TODO Auto-generated method stub
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ // we're using the md5 as identifier
+ GridFSDBFile f = fs.findOne(new BasicDBObject("md5", identifier.toString()));
+ if (f != null) {
+ return new MongoDataRecord(identifier, f);
+ } else {
+ return null;
+ }
}
public DataRecord getRecord(DataIdentifier identifier) throws DataStoreException {
- return null; // TODO Auto-generated method stub
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ DataRecord record = getRecordIfStored(identifier);
+ if (record == null) {
+ throw new DataStoreException("Record not found: " + identifier);
+ }
+ return record;
}
public DataRecord addRecord(InputStream stream) throws DataStoreException {
- return null; // TODO Auto-generated method stub
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ GridFSInputFile f = fs.createFile(stream);
+ try {
+ // saveChunks() computes the md5 but doesn't persist the data
+ f.saveChunks();
+ } catch (IOException e) {
+ throw new DataStoreException("failed to read from stream", e);
+ }
+ String md5 = f.getMD5();
+
+ // check server before uploading new file
+ GridFSDBFile dbf = fs.findOne(new BasicDBObject("md5", md5));
+ if (dbf == null) {
+ // persist the data
+ f.save();
+ dbf = fs.findOne(new BasicDBObject("md5", md5));
+ }
+ return new MongoDataRecord(new DataIdentifier(dbf.getMD5()), dbf);
}
- public void updateModifiedDateOnAccess(long before) {
- // TODO Auto-generated method stub
+ public int getMinRecordLength() {
+ return minRecordLength;
}
- public int deleteAllOlderThan(long min) throws DataStoreException {
- return 0; // TODO Auto-generated method stub
+ public void updateModifiedDateOnAccess(long before) {
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ minModifiedDate = before;
}
public Iterator<DataIdentifier> getAllIdentifiers() throws DataStoreException {
- return null; // TODO Auto-generated method stub
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ ArrayList<DataIdentifier> results = new ArrayList<DataIdentifier>();
+
+ DBCursor cursor = fs.getFileList();
+ try {
+ while (cursor.hasNext()) {
+ GridFSFile f = (GridFSFile) cursor.next();
+ // we're using the md5 as identifier
+ results.add(new DataIdentifier(f.getMD5()));
+ }
+ return results.iterator();
+ } finally {
+ cursor.close();
+ }
}
- public int getMinRecordLength() {
- return 0; // TODO Auto-generated method stub
+ public int deleteAllOlderThan(long min) throws DataStoreException {
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ // TODO implement GC support
+ return 0;
}
public void clearInUse() {
- // TODO Auto-generated method stub
+ if (!initialized) {
+ throw new IllegalStateException("not initialized");
+ }
+
+ // TODO implement GC support
}
+
+ //-------------------------------------------------------< implementation >
}
Modified: jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoPersistenceManager.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoPersistenceManager.java?rev=1299437&r1=1299436&r2=1299437&view=diff
==============================================================================
--- jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoPersistenceManager.java (original)
+++ jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/MongoPersistenceManager.java Sun Mar 11 20:01:51 2012
@@ -50,6 +50,8 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.jackrabbit.core.persistence.mongo.Utils.encodeName;
+
/**
*
*/
@@ -439,67 +441,6 @@ public class MongoPersistenceManager ext
return (refs.findOne(key) != null);
}
- //-------------------------------------------------------< implementation >
-
- protected final static String ENCODED_DOT = "_x46_";
- protected final static String ENCODED_DOLLAR_SIGN = "_x36_";
-
- /**
- * see http://www.mongodb.org/display/DOCS/Legal+Key+Names
- *
- * @param name
- * @return
- */
- protected static String encodeName(String name) {
- StringBuilder buf = null;
- for (int i = 0; i < name.length(); i++) {
- if (i == 0 && name.charAt(i) == '$') {
- // mongodb field names must not start with '$'
- buf = new StringBuilder();
- buf.append(ENCODED_DOLLAR_SIGN);
- } else if (name.charAt(i) == '.') {
- // . is a reserved char for mongodb field names
- if (buf == null) {
- buf = new StringBuilder(name.substring(0, i));
- }
- buf.append(ENCODED_DOT);
- } else {
- if (buf != null) {
- buf.append(name.charAt(i));
- }
- }
- }
-
- return buf == null ? name : buf.toString();
- }
-
- protected static String decodeName(String name) {
- StringBuilder buf = null;
-
- int lastPos = 0;
- if (name.startsWith(ENCODED_DOLLAR_SIGN)) {
- buf = new StringBuilder("$");
- lastPos = ENCODED_DOLLAR_SIGN.length();
- }
-
- int pos;
- while ((pos = name.indexOf(ENCODED_DOT, lastPos)) != -1) {
- if (buf == null) {
- buf = new StringBuilder();
- }
- buf.append(name.substring(lastPos, pos));
- buf.append('.');
- lastPos = pos + ENCODED_DOT.length();
- }
-
- if (buf != null) {
- buf.append(name.substring(lastPos));
- return buf.toString();
- } else {
- return name;
- }
- }
-
//--------------------------------------------------------< inner classes >
protected class MongoBlobStore implements BLOBStore {
Added: jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/Utils.java?rev=1299437&view=auto
==============================================================================
--- jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/Utils.java (added)
+++ jackrabbit/sandbox/jackrabbit-mongo-persistence/src/main/java/org/apache/jackrabbit/core/persistence/mongo/Utils.java Sun Mar 11 20:01:51 2012
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.core.persistence.mongo;
+
+/**
+ *
+ */
+public final class Utils {
+ private final static String ENCODED_DOT = "_x46_";
+ private final static String ENCODED_DOLLAR_SIGN = "_x36_";
+
+ /**
+ * see http://www.mongodb.org/display/DOCS/Legal+Key+Names
+ *
+ * @param name
+ * @return
+ */
+ public static String encodeName(String name) {
+ StringBuilder buf = null;
+ for (int i = 0; i < name.length(); i++) {
+ if (i == 0 && name.charAt(i) == '$') {
+ // mongodb field names must not start with '$'
+ buf = new StringBuilder();
+ buf.append(ENCODED_DOLLAR_SIGN);
+ } else if (name.charAt(i) == '.') {
+ // . is a reserved char for mongodb field names
+ if (buf == null) {
+ buf = new StringBuilder(name.substring(0, i));
+ }
+ buf.append(ENCODED_DOT);
+ } else {
+ if (buf != null) {
+ buf.append(name.charAt(i));
+ }
+ }
+ }
+
+ return buf == null ? name : buf.toString();
+ }
+
+ public static String decodeName(String name) {
+ StringBuilder buf = null;
+
+ int lastPos = 0;
+ if (name.startsWith(ENCODED_DOLLAR_SIGN)) {
+ buf = new StringBuilder("$");
+ lastPos = ENCODED_DOLLAR_SIGN.length();
+ }
+
+ int pos;
+ while ((pos = name.indexOf(ENCODED_DOT, lastPos)) != -1) {
+ if (buf == null) {
+ buf = new StringBuilder();
+ }
+ buf.append(name.substring(lastPos, pos));
+ buf.append('.');
+ lastPos = pos + ENCODED_DOT.length();
+ }
+
+ if (buf != null) {
+ buf.append(name.substring(lastPos));
+ return buf.toString();
+ } else {
+ return name;
+ }
+ }
+}
+