You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by le...@apache.org on 2014/06/04 18:36:19 UTC
[02/50] [abbrv] git commit: First round of update of MongoStore after
GORA_94 merge
First round of update of MongoStore after GORA_94 merge
Project: http://git-wip-us.apache.org/repos/asf/gora/repo
Commit: http://git-wip-us.apache.org/repos/asf/gora/commit/ab2102a2
Tree: http://git-wip-us.apache.org/repos/asf/gora/tree/ab2102a2
Diff: http://git-wip-us.apache.org/repos/asf/gora/diff/ab2102a2
Branch: refs/heads/master
Commit: ab2102a29ced91101c8d4987115996071efd02f9
Parents: 30c671b
Author: Damien Raude-Morvan <da...@dictanova.com>
Authored: Wed Apr 16 00:18:47 2014 +0200
Committer: Damien Raude-Morvan <da...@dictanova.com>
Committed: Wed Apr 16 00:18:55 2014 +0200
----------------------------------------------------------------------
gora-mongodb/pom.xml | 2 +-
.../apache/gora/mongodb/query/MongoDBQuery.java | 5 ++-
.../gora/mongodb/query/MongoDBResult.java | 1 -
.../apache/gora/mongodb/store/MongoStore.java | 46 ++++++++------------
.../src/test/conf/gora-mongodb-mapping.xml | 1 +
5 files changed, 24 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/pom.xml
----------------------------------------------------------------------
diff --git a/gora-mongodb/pom.xml b/gora-mongodb/pom.xml
index 5ff0b21..f48270c 100644
--- a/gora-mongodb/pom.xml
+++ b/gora-mongodb/pom.xml
@@ -119,7 +119,7 @@
</dependency>
<dependency>
- <groupId>org.apache.hadoop</groupId>
+ <groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java
----------------------------------------------------------------------
diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java
index ec440da..81fe765 100644
--- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java
+++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBQuery.java
@@ -71,7 +71,10 @@ public class MongoDBQuery<K, T extends PersistentBase> extends QueryBase<K, T> {
BasicDBObject proj = new BasicDBObject();
for (String k : fields) {
- proj.put(mapping.getDocumentField(k), true);
+ String dbFieldName = mapping.getDocumentField(k);
+ if (dbFieldName != null && dbFieldName.length() > 0) {
+ proj.put(dbFieldName, true);
+ }
}
return proj;
http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
----------------------------------------------------------------------
diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
index b3550ef..d1c53e7 100644
--- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
+++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/query/MongoDBResult.java
@@ -75,7 +75,6 @@ public class MongoDBResult<K, T extends PersistentBase> extends ResultBase<K, T>
persistent = ((MongoStore<K, T>) getDataStore()).newInstance(obj,
getQuery().getFields());
if (persistent != null) {
- persistent.clearNew();
persistent.clearDirty();
}
http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java
----------------------------------------------------------------------
diff --git a/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java b/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java
index e94113a..3313469 100644
--- a/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java
+++ b/gora-mongodb/src/main/java/org/apache/gora/mongodb/store/MongoStore.java
@@ -37,13 +37,11 @@ import org.apache.gora.mongodb.utils.BSONDecorator;
import org.apache.gora.mongodb.utils.GoraDBEncoder;
import org.apache.gora.mongodb.query.MongoDBQuery;
import org.apache.gora.mongodb.query.MongoDBResult;
-import org.apache.gora.persistency.ListGenericArray;
+import org.apache.gora.persistency.impl.DirtyListWrapper;
import org.apache.gora.persistency.Persistent;
-import org.apache.gora.persistency.StateManager;
-import org.apache.gora.persistency.StatefulHashMap;
+import org.apache.gora.persistency.impl.DirtyMapWrapper;
import org.apache.gora.persistency.impl.BeanFactoryImpl;
import org.apache.gora.persistency.impl.PersistentBase;
-import org.apache.gora.persistency.impl.StateManagerImpl;
import org.apache.gora.query.PartitionQuery;
import org.apache.gora.query.Query;
import org.apache.gora.query.Result;
@@ -353,7 +351,6 @@ public class MongoStore<K, T extends PersistentBase> extends
// Build the corresponding persistent and clears its states
final T persistent = newInstance(res, fields);
if (persistent != null) {
- persistent.clearNew();
persistent.clearDirty();
}
// Done
@@ -371,7 +368,7 @@ public class MongoStore<K, T extends PersistentBase> extends
@Override
public void put(K key, T obj) {
// Save the object in the database
- if (obj.isNew() || obj.isDirty()) {
+ if (obj.isDirty()) {
putUpdate(key, obj);
// TODO: why is Nutch marking objects as new ?
// putInsert(key, obj);
@@ -381,7 +378,6 @@ public class MongoStore<K, T extends PersistentBase> extends
LOG.info("Ignored putting object {} in the store as it is neither "
+ "new, neither dirty.", new Object[] { obj });
// Clear its state
- obj.clearNew();
obj.clearDirty();
}
}
@@ -524,7 +520,6 @@ public class MongoStore<K, T extends PersistentBase> extends
BSONDecorator easybson = new BSONDecorator(obj);
// Create new empty persistent bean instance
T persistent = newPersistent();
- StateManager stateManager = persistent.getStateManager();
fields = getFieldsToQuery(fields);
// Populate each field
@@ -546,7 +541,7 @@ public class MongoStore<K, T extends PersistentBase> extends
Object result = fromDBObject(fieldSchema, storeType, field, docf, easybson);
persistent.put(field.pos(), result);
}
- stateManager.clearDirty(persistent);
+ persistent.clearDirty();
return persistent;
}
@@ -555,7 +550,7 @@ public class MongoStore<K, T extends PersistentBase> extends
switch (fieldSchema.getType()) {
case MAP:
BasicDBObject map = easybson.getDBObject(docf);
- StatefulHashMap<Utf8, Object> rmap = new StatefulHashMap<Utf8, Object>();
+ Map<Utf8, Object> rmap = new HashMap<Utf8, Object>();
for (Entry<String, Object> e : map.entrySet()) {
// ensure Key decoding -> middle dots replaced with dots
// FIXME: better approach ?
@@ -573,31 +568,28 @@ public class MongoStore<K, T extends PersistentBase> extends
break;
}
}
- rmap.clearStates();
- result = rmap;
+ result = new DirtyMapWrapper(rmap);
break;
case ARRAY:
List<Object> list = easybson.getDBList(docf);
switch (fieldSchema.getElementType().getType()) {
case STRING:
- ListGenericArray<Utf8> arrS = new ListGenericArray<Utf8>(fieldSchema);
+ List<Utf8> arrS = new ArrayList<Utf8>();
for (Object o : list)
arrS.add(new Utf8((String) o));
- result = arrS;
+ result = new DirtyListWrapper<Utf8>(arrS);
break;
case BYTES:
- ListGenericArray<ByteBuffer> arrB = new ListGenericArray<ByteBuffer>(
- fieldSchema);
+ List<ByteBuffer> arrB = new ArrayList<ByteBuffer>();
for (Object o : list)
arrB.add(ByteBuffer.wrap((byte[]) o));
- result = arrB;
+ result = new DirtyListWrapper<ByteBuffer>(arrB);
break;
default:
- ListGenericArray<Object> arrT = new ListGenericArray<Object>(
- fieldSchema);
+ List<Object> arrT = new ArrayList<Object>();
for (Object o : list)
arrT.add(o);
- result = arrT;
+ result = new DirtyListWrapper<Object>(arrT);
break;
}
break;
@@ -756,8 +748,7 @@ public class MongoStore<K, T extends PersistentBase> extends
private BasicDBObject newUpdateSetInstance(T persistent) {
BasicDBObject result = new BasicDBObject();
for (Field f : persistent.getSchema().getFields()) {
- if (persistent.isReadable(f.pos()) && persistent.isDirty(f.pos())
- && (persistent.get(f.pos()) != null)) {
+ if (persistent.isDirty(f.pos()) && (persistent.get(f.pos()) != null)) {
String docf = mapping.getDocumentField(f.name());
Object value = persistent.get(f.pos());
DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
@@ -791,8 +782,7 @@ public class MongoStore<K, T extends PersistentBase> extends
private BasicDBObject newUpdateUnsetInstance(T persistent) {
BasicDBObject result = new BasicDBObject();
for (Field f : persistent.getSchema().getFields()) {
- if (persistent.isReadable(f.pos()) && persistent.isDirty(f.pos())
- && (persistent.get(f.pos()) == null)) {
+ if (persistent.isDirty(f.pos()) && (persistent.get(f.pos()) == null)) {
String docf = mapping.getDocumentField(f.name());
Object value = persistent.get(f.pos());
DocumentFieldType storeType = mapping.getDocumentFieldType(docf);
@@ -825,7 +815,7 @@ public class MongoStore<K, T extends PersistentBase> extends
+ ": To store a Gora 'array', target Mongo mapping have to be of 'list' type");
}
Schema elementSchema = fieldSchema.getElementType();
- result = toMongoList((GenericArray<?>) value, elementSchema.getType());
+ result = toMongoList((List<?>) value, elementSchema.getType());
break;
case BYTES:
// Beware of ByteBuffer not being safely serialized
@@ -957,7 +947,7 @@ public class MongoStore<K, T extends PersistentBase> extends
case ARRAY:
easybson.put(
key,
- toMongoList((GenericArray<?>) value, field.schema().getElementType()
+ toMongoList((List<?>) value, field.schema().getElementType()
.getType()));
break;
case BYTES:
@@ -990,7 +980,7 @@ public class MongoStore<K, T extends PersistentBase> extends
case ARRAY:
record.put(
member.name(),
- toMongoList((GenericArray<?>) recValue, member.schema()
+ toMongoList((List<?>) recValue, member.schema()
.getElementType().getType()));
break;
case LONG:
@@ -1077,7 +1067,7 @@ public class MongoStore<K, T extends PersistentBase> extends
* @return a {@link BasicDBList} version of the {@link GenericArray} that can
* be safely serialized into MongoDB.
*/
- private BasicDBList toMongoList(GenericArray<?> array, Type type) {
+ private BasicDBList toMongoList(Collection<?> array, Type type) {
// Handle null case
if (array == null)
return null;
http://git-wip-us.apache.org/repos/asf/gora/blob/ab2102a2/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml
----------------------------------------------------------------------
diff --git a/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml b/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml
index a99a12b..b89c888 100644
--- a/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml
+++ b/gora-mongodb/src/test/conf/gora-mongodb-mapping.xml
@@ -31,6 +31,7 @@
<field name="content" docfield="content" type="binary"/>
<field name="parsedContent" docfield="pContent" type="list"/>
<field name="outlinks" docfield="links.out" type="document"/>
+ <field name="headers" docfield="headers" type="document"/>
<field name="metadata" docfield="metadata" type="document"/>
</class>