You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@rya.apache.org by ca...@apache.org on 2017/08/07 02:56:38 UTC
incubator-rya git commit: RYA-313. Closes #196. Aggregation now is
performed over batches of 1000.
Repository: incubator-rya
Updated Branches:
refs/heads/master 2564ac0ab -> 1d99212e6
RYA-313. Closes #196.
Aggregation now is performed over batches of 1000.
Project: http://git-wip-us.apache.org/repos/asf/incubator-rya/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-rya/commit/1d99212e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-rya/tree/1d99212e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-rya/diff/1d99212e
Branch: refs/heads/master
Commit: 1d99212e60ac1ac0a0c8fef227e5c959c6826aa1
Parents: 2564ac0
Author: isper3at <sm...@gmail.com>
Authored: Thu Aug 3 19:03:11 2017 -0400
Committer: Caleb Meier <ca...@parsons.com>
Committed: Sun Aug 6 19:54:39 2017 -0700
----------------------------------------------------------------------
.../apache/rya/mongodb/MongoDBQueryEngine.java | 29 +++++--
.../operators/aggregation/AggregationUtil.java | 14 ++--
.../aggregation/PipelineOperators.java | 23 +-----
.../operators/aggregation/SetOperators.java | 87 ++++----------------
.../aggregation/VariableOperators.java | 37 ++-------
.../operators/query/ArrayOperators.java | 24 ++----
.../operators/query/ComparisonOperators.java | 47 ++---------
.../operators/query/ConditionalOperators.java | 69 +++-------------
.../operators/query/LogicalOperators.java | 55 +++----------
.../RyaStatementBindingSetCursorIterator.java | 33 ++++----
.../iter/RyaStatementCursorIterator.java | 33 ++++----
11 files changed, 133 insertions(+), 318 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java
index 2b49e61..8932fc4 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/MongoDBQueryEngine.java
@@ -20,7 +20,6 @@ package org.apache.rya.mongodb;
import static com.google.common.base.Preconditions.checkNotNull;
-import java.io.Closeable;
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
@@ -39,6 +38,7 @@ import org.apache.rya.mongodb.iter.NonCloseableRyaStatementCursorIterator;
import org.apache.rya.mongodb.iter.RyaStatementBindingSetCursorIterator;
import org.apache.rya.mongodb.iter.RyaStatementCursorIterable;
import org.apache.rya.mongodb.iter.RyaStatementCursorIterator;
+import org.bson.Document;
import org.calrissian.mango.collect.CloseableIterable;
import org.openrdf.query.BindingSet;
@@ -48,6 +48,8 @@ import com.mongodb.DB;
import com.mongodb.DBCollection;
import com.mongodb.DBObject;
import com.mongodb.MongoClient;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.client.MongoDatabase;
import info.aduna.iteration.CloseableIteration;
@@ -91,13 +93,17 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio
final Set<DBObject> queries = new HashSet<DBObject>();
final DBObject query = strategy.getQuery(stmt);
queries.add(query);
- final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(coll, queries, strategy, conf.getAuthorizations());
+ final MongoDatabase db = mongoClient.getDatabase(conf.getMongoDBName());
+ final MongoCollection<Document> collection = db.getCollection(conf.getTriplesCollectionName());
+ final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(collection, queries, strategy,
+ conf.getAuthorizations());
if (maxResults != null) {
iterator.setMaxResults(maxResults);
}
return iterator;
}
+
@Override
public CloseableIteration<? extends Entry<RyaStatement, BindingSet>, RyaDAOException> queryWithBindingSet(
final Collection<Entry<RyaStatement, BindingSet>> stmts,
@@ -118,7 +124,8 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio
}
// TODO not sure what to do about regex ranges?
- final RyaStatementBindingSetCursorIterator iterator = new RyaStatementBindingSetCursorIterator(coll, rangeMap, strategy, conf.getAuthorizations());
+ final RyaStatementBindingSetCursorIterator iterator = new RyaStatementBindingSetCursorIterator(
+ getCollection(conf), rangeMap, strategy, conf.getAuthorizations());
if (maxResults != null) {
iterator.setMaxResults(maxResults);
@@ -145,7 +152,8 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio
}
// TODO not sure what to do about regex ranges?
- final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(coll, queries, strategy, configuration.getAuthorizations());
+ final RyaStatementCursorIterator iterator = new RyaStatementCursorIterator(getCollection(conf), queries,
+ strategy, configuration.getAuthorizations());
if (maxResults != null) {
iterator.setMaxResults(maxResults);
@@ -166,7 +174,9 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio
// TODO not sure what to do about regex ranges?
// TODO this is gross
- final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable(new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(coll, queries, strategy, configuration.getAuthorizations())));
+ final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable(
+ new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(getCollection(getConf()),
+ queries, strategy, configuration.getAuthorizations())));
return iterator;
} catch (final Exception e) {
@@ -185,7 +195,9 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio
// TODO not sure what to do about regex ranges?
// TODO this is gross
- final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable(new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(coll, queries, strategy, configuration.getAuthorizations())));
+ final RyaStatementCursorIterable iterator = new RyaStatementCursorIterable(
+ new NonCloseableRyaStatementCursorIterator(new RyaStatementCursorIterator(getCollection(getConf()),
+ queries, strategy, configuration.getAuthorizations())));
return iterator;
} catch (final Exception e) {
@@ -193,6 +205,11 @@ public class MongoDBQueryEngine implements RyaQueryEngine<MongoDBRdfConfiguratio
}
}
+ private MongoCollection getCollection(final MongoDBRdfConfiguration conf) {
+ final MongoDatabase db = mongoClient.getDatabase(conf.getMongoDBName());
+ return db.getCollection(conf.getTriplesCollectionName());
+ }
+
@Override
public void close() throws IOException {
// if (mongoClient != null){ mongoClient.close(); }
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java
index 6e4f250..082d7bd 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/AggregationUtil.java
@@ -34,9 +34,9 @@ import org.apache.rya.mongodb.MongoDbRdfConstants;
import org.apache.rya.mongodb.dao.SimpleMongoDBStorageStrategy;
import org.apache.rya.mongodb.document.operators.aggregation.PipelineOperators.RedactAggregationResult;
import org.apache.rya.mongodb.document.util.AuthorizationsUtil;
+import org.bson.Document;
import com.google.common.collect.Lists;
-import com.mongodb.BasicDBObject;
import com.mongodb.DBObject;
/**
@@ -58,7 +58,7 @@ public final class AggregationUtil {
* @return the {@link List} of {@link DBObject}s that represents the $redact
* aggregation pipeline.
*/
- public static List<DBObject> createRedactPipeline(final Authorizations authorizations) {
+ public static List<Document> createRedactPipeline(final Authorizations authorizations) {
if (MongoDbRdfConstants.ALL_AUTHORIZATIONS.equals(authorizations)) {
return Lists.newArrayList();
}
@@ -69,29 +69,29 @@ public final class AggregationUtil {
final String mapVariableCursorName = "dvItemCursorTag";
- final BasicDBObject anyElementTrue =
+ final Document anyElementTrue =
anyElementTrue(
map(
documentVisibilityField,
mapVariableCursorName,
setIsSubsetNullSafe(
"$$" + mapVariableCursorName,
- authList.toArray()
+ authList
)
)
);
// If the field is empty then there are no authorizations required,
// so all users should be able to view it when they query.
- final BasicDBObject isFieldSizeZero =
+ final Document isFieldSizeZero =
eq(
size(documentVisibilityField),
0
);
- final BasicDBObject orExpression = or(anyElementTrue, isFieldSizeZero);
+ final Document orExpression = or(anyElementTrue, isFieldSizeZero);
- final List<DBObject> pipeline = new ArrayList<>();
+ final List<Document> pipeline = new ArrayList<>();
pipeline.add(
redact(
orExpression,
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java
index b73a6c7..2d65312 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/PipelineOperators.java
@@ -21,6 +21,8 @@ package org.apache.rya.mongodb.document.operators.aggregation;
import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.rya.mongodb.document.operators.query.ConditionalOperators.cond;
+import org.bson.Document;
+
import com.mongodb.BasicDBObject;
import com.mongodb.BasicDBObjectBuilder;
@@ -83,26 +85,9 @@ public final class PipelineOperators {
* the expression passes.
* @param rejectResult the {@link RedactAggregationResult} to return when
* the expression fails.
- * @return the $redact expression {@link BasicDBObject}.
- */
- public static BasicDBObject redact(final BasicDBObject expression, final RedactAggregationResult acceptResult, final RedactAggregationResult rejectResult) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) redact(builder, expression, acceptResult, rejectResult).get();
- }
-
- /**
- * Creates a $redact expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param expression the expression to run redact on.
- * @param acceptResult the {@link RedactAggregationResult} to return when
- * the expression passes.
- * @param rejectResult the {@link RedactAggregationResult} to return when
- * the expression fails.
* @return the $redact expression {@link BasicDBObjectBuilder}.
*/
- public static BasicDBObjectBuilder redact(final BasicDBObjectBuilder builder, final BasicDBObject expression, final RedactAggregationResult acceptResult, final RedactAggregationResult rejectResult) {
- checkNotNull(builder);
- builder.add("$redact", cond(expression, acceptResult.toString(), rejectResult.toString()));
- return builder;
+ public static Document redact(final Document expression, final RedactAggregationResult acceptResult, final RedactAggregationResult rejectResult) {
+ return new Document("$redact", cond(expression, acceptResult.toString(), rejectResult.toString()));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java
index d5b8738..9ef8204 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/SetOperators.java
@@ -18,15 +18,13 @@
*/
package org.apache.rya.mongodb.document.operators.aggregation;
-import static com.google.common.base.Preconditions.checkNotNull;
import static org.apache.rya.mongodb.document.operators.query.ConditionalOperators.ifNull;
import java.util.Arrays;
import java.util.Collections;
+import java.util.List;
-import com.mongodb.BasicDBObject;
-import com.mongodb.BasicDBObjectBuilder;
-import com.mongodb.DBObject;
+import org.bson.Document;
/**
* Utility methods for MongoDB set operators.
@@ -42,48 +40,20 @@ public final class SetOperators {
* Checks if the field intersects the set.
* @param field the field to check.
* @param set the set to check.
- * @return the $setIntersection expression {@link BasicDBObject}.
+ * @return the $setIntersection expression {@link Document}.
*/
- public static BasicDBObject setIntersection(final String field, final Object[] set) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) setIntersection(builder, field, set).get();
- }
-
- /**
- * Checks if the field intersects the set.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param field the field to check.
- * @param set the set to check.
- * @return the $setIntersection expression {@link BasicDBObjectBuilder}.
- */
- public static BasicDBObjectBuilder setIntersection(final BasicDBObjectBuilder builder, final String field, final Object[] set) {
- checkNotNull(builder);
- builder.add("$setIntersection", Arrays.asList(field, set));
- return builder;
- }
-
- /**
- * Checks if the expression is a subset of the set.
- * @param expression the expression to see if it's in the set.
- * @param set the set to check against.
- * @return the $setIsSubset expression {@link BasicDBObject}.
- */
- public static BasicDBObject setIsSubset(final DBObject expression, final Object[] set) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) setIsSubset(builder, expression, set).get();
+ public static Document setIntersection(final String field, final List<String> set) {
+ return new Document("$setIntersection", Arrays.asList(field, set));
}
/**
* Checks if the expression is a subset of the set.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
* @param expression the expression to see if it's in the set.
* @param set the set to check against.
- * @return the $setIsSubset expression {@link BasicDBObjectBuilder}.
+ * @return the $setIsSubset expression {@link Document}.
*/
- public static BasicDBObjectBuilder setIsSubset(final BasicDBObjectBuilder builder, final DBObject expression, final Object[] set) {
- checkNotNull(builder);
- builder.add("$setIsSubset", Arrays.asList(expression, set).toArray(new Object[0]));
- return builder;
+ public static Document setIsSubset(final Document expression, final List<String> set) {
+ return new Document("$setIsSubset", Arrays.asList(expression, set));
}
/**
@@ -91,25 +61,11 @@ public final class SetOperators {
* null.
* @param field the field to see if it's in the set.
* @param set the set to check against.
- * @return the $setIsSubset expression {@link BasicDBObject}.
+ * @return the $setIsSubset expression {@link Document}.
*/
- public static BasicDBObject setIsSubsetNullSafe(final String field, final Object[] set) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) setIsSubsetNullSafe(builder, field, set).get();
- }
-
- /**
- * Checks if the field is a subset of the set and is safe if the field is
- * null.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param field the field to see if it's in the set.
- * @param set the set to check against.
- * @return the $setIsSubset expression {@link BasicDBObjectBuilder}.
- */
- public static BasicDBObjectBuilder setIsSubsetNullSafe(final BasicDBObjectBuilder builder, final String field, final Object[] set) {
- checkNotNull(builder);
- final Object emptyAccess = Collections.emptyList().toArray();
- return setIsSubset(builder,
+ public static Document setIsSubsetNullSafe(final String field, final List<String> set) {
+ final Object emptyAccess = Collections.emptyList();
+ return setIsSubset(
ifNull(
field,
emptyAccess
@@ -121,22 +77,9 @@ public final class SetOperators {
/**
* Checks if any elements from the expression are {@code true}.
* @param expression the expression to see if any elements are {@code true}.
- * @return the $anyElementTrue expression {@link BasicDBObject}.
- */
- public static BasicDBObject anyElementTrue(final DBObject expression) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) anyElementTrue(builder, expression).get();
- }
-
- /**
- * Checks if any elements from the expression are {@code true}.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param expression the expression to see if any elements are {@code true}.
- * @return the $anyElementTrue expression {@link BasicDBObjectBuilder}.
+ * @return the $anyElementTrue expression {@link Document}.
*/
- public static BasicDBObjectBuilder anyElementTrue(final BasicDBObjectBuilder builder, final DBObject expression) {
- checkNotNull(builder);
- builder.add("$anyElementTrue", expression);
- return builder;
+ public static Document anyElementTrue(final Document expression) {
+ return new Document("$anyElementTrue", expression);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java
index 1fcf8e3..e014dc3 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/aggregation/VariableOperators.java
@@ -18,11 +18,9 @@
*/
package org.apache.rya.mongodb.document.operators.aggregation;
-import static com.google.common.base.Preconditions.checkNotNull;
+import javax.xml.parsers.DocumentBuilder;
-import com.mongodb.BasicDBObject;
-import com.mongodb.BasicDBObjectBuilder;
-import com.mongodb.DBObject;
+import org.bson.Document;
/**
* Utility methods for variable operators.
@@ -43,31 +41,14 @@ public final class VariableOperators {
* by this variable.
* @param in the expression to apply to each item in the {@code input}
* array. The expression accesses the item by its variable name.
- * @return the $map expression {@link BasicDBObject}.
+ * @return the $map expression {@link DocumentBuilder}.
*/
- public static BasicDBObject map(final String input, final String as, final DBObject in) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) map(builder, input, as, in).get();
- }
+ public static Document map(final String input, final String as, final Document in) {
+ final Document mapDoc = new Document()
+ .append("input", input)
+ .append("as", as)
+ .append("in", in);
- /**
- * Applies an expression to each item in an array and returns an array with
- * the applied results.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param input an expression that resolves to an array.
- * @param as the variable name for the items in the {@code input} array.
- * The {@code in} expression accesses each item in the {@code input} array
- * by this variable.
- * @param in the expression to apply to each item in the {@code input}
- * array. The expression accesses the item by its variable name.
- * @return the $map expression {@link BasicDBObjectBuilder}.
- */
- public static BasicDBObjectBuilder map(final BasicDBObjectBuilder builder, final String input, final String as, final DBObject in) {
- checkNotNull(builder);
- builder.push("$map")
- .add("input", input)
- .add("as", as)
- .add("in", in);
- return builder;
+ return new Document("$map", mapDoc);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java
index 1413ce8..b3daa7d 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ArrayOperators.java
@@ -18,10 +18,9 @@
*/
package org.apache.rya.mongodb.document.operators.query;
-import static com.google.common.base.Preconditions.checkNotNull;
+import javax.xml.parsers.DocumentBuilder;
-import com.mongodb.BasicDBObject;
-import com.mongodb.BasicDBObjectBuilder;
+import org.bson.Document;
/**
* Utility methods for array operators.
@@ -36,22 +35,9 @@ public final class ArrayOperators {
/**
* Creates an $size MongoDB expression.
* @param expression the expression to get the size of.
- * @return the $size expression {@link BasicDBObject}.
+ * @return the $size expression {@link DocumentBuilder}.
*/
- public static BasicDBObject size(final Object expression) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) size(builder, expression).get();
- }
-
- /**
- * Creates an $size MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param expression the expression to get the size of.
- * @return the $size expression {@link BasicDBObjectBuilder}.
- */
- public static BasicDBObjectBuilder size(final BasicDBObjectBuilder builder, final Object expression) {
- checkNotNull(builder);
- builder.add("$size", expression);
- return builder;
+ public static Document size(final Object expression) {
+ return new Document("$size", expression);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java
index 596493a..5187401 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ComparisonOperators.java
@@ -18,12 +18,9 @@
*/
package org.apache.rya.mongodb.document.operators.query;
-import static com.google.common.base.Preconditions.checkNotNull;
-
import java.util.Arrays;
-import com.mongodb.BasicDBObject;
-import com.mongodb.BasicDBObjectBuilder;
+import org.bson.Document;
/**
* Utility methods for comparison operators.
@@ -37,49 +34,23 @@ public final class ComparisonOperators {
/**
* Creates a $gt MongoDB expression.
+ *
* @param expression the expression.
* @param value the value to test if the expression is greater than
- * @return the $gt expression {@link BasicDBObject}.
- */
- public static BasicDBObject gt(final BasicDBObject expression, final Number value) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) gt(builder, expression, value).get();
- }
-
- /**
- * Creates a $gt MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param expression the expression.
- * @param value the value to test if the expression is greater than
- * @return the $gt expression {@link BasicDBObjectBuilder}.
- */
- public static BasicDBObjectBuilder gt(final BasicDBObjectBuilder builder, final BasicDBObject expression, final Number value) {
- checkNotNull(builder);
- builder.add("$gt", Arrays.asList(expression, value));
- return builder;
- }
-
- /**
- * Creates an $eq MongoDB expression.
- * @param expression1 the first expression.
- * @param expression2 the second expression.
- * @return the $eq expression {@link BasicDBObject}.
+ * @return the $gt expression {@link Document}.
*/
- public static BasicDBObject eq(final BasicDBObject expression1, final Object expression2) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) eq(builder, expression1, expression2).get();
+ public static Document gt(final Document expression, final Number value) {
+ return new Document("$gt", Arrays.asList(expression, value));
}
/**
* Creates an $eq MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
+ *
* @param expression1 the first expression.
* @param expression2 the second expression.
- * @return the $eq expression {@link BasicDBObjectBuilder}.
+ * @return the $eq expression {@link Document}.
*/
- public static BasicDBObjectBuilder eq(final BasicDBObjectBuilder builder, final BasicDBObject expression1, final Object expression2) {
- checkNotNull(builder);
- builder.add("$eq", Arrays.asList(expression1, expression2));
- return builder;
+ public static Document eq(final Document expression1, final Object expression2) {
+ return new Document("$eq", Arrays.asList(expression1, expression2));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java
index c976519..dd83762 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/ConditionalOperators.java
@@ -18,10 +18,10 @@
*/
package org.apache.rya.mongodb.document.operators.query;
-import static com.google.common.base.Preconditions.checkNotNull;
-
import java.util.Arrays;
+import org.bson.Document;
+
import com.mongodb.BasicDBObject;
import com.mongodb.BasicDBObjectBuilder;
@@ -42,74 +42,27 @@ public final class ConditionalOperators {
* {@code ifStatement} is {@code true}.
* @param elseResult the {@link Object} to return when the
* {@code ifStatement} is {@code false}.
- * @return the "if" expression {@link BasicDBObject}.
- */
- public static BasicDBObject ifThenElse(final BasicDBObject ifStatement, final Object thenResult, final Object elseResult) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) ifThenElse(builder, ifStatement, thenResult, elseResult).get();
- }
-
- /**
- * Creates an "if-then-else" MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param ifStatement the "if" statement {@link BasicDBObject}.
- * @param thenResult the {@link Object} to return when the
- * {@code ifStatement} is {@code true}.
- * @param elseResult the {@link Object} to return when the
- * {@code ifStatement} is {@code false}.
* @return the "if" expression {@link BasicDBObjectBuilder}.
*/
- public static BasicDBObjectBuilder ifThenElse(final BasicDBObjectBuilder builder, final BasicDBObject ifStatement, final Object thenResult, final Object elseResult) {
- checkNotNull(builder);
- builder.add("if", ifStatement);
- builder.append("then", thenResult);
- builder.append("else", elseResult);
- return builder;
- }
-
- /**
- * Checks if the expression is {@code null} and replaces it if it is.
- * @param expression the expression to {@code null} check.
- * @param replacementExpression the expression to replace it with if it's
- * {@code null}.
- * @return the $ifNull expression {@link BasicDBObject}.
- */
- public static BasicDBObject ifNull(final Object expression, final Object replacementExpression) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) ifNull(builder, expression, replacementExpression).get();
+ public static Document ifThenElse(final Document ifStatement, final Object thenResult, final Object elseResult) {
+ return new Document("if", ifStatement)
+ .append("then", thenResult)
+ .append("else", elseResult);
}
/**
* Checks if the expression is {@code null} and replaces it if it is.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
* @param expression the expression to {@code null} check.
* @param replacementExpression the expression to replace it with if it's
* {@code null}.
* @return the $ifNull expression {@link BasicDBObjectBuilder}.
*/
- public static BasicDBObjectBuilder ifNull(final BasicDBObjectBuilder builder, final Object expression, final Object replacementExpression) {
- checkNotNull(builder);
- builder.add("$ifNull", Arrays.asList(expression, replacementExpression));
- return builder;
- }
-
- /**
- * Creates an "$cond" MongoDB expression.
- * @param expression the expression {@link BasicDBObject}.
- * @param thenResult the {@link Object} to return when the
- * {@code expression} is {@code true}.
- * @param elseResult the {@link Object} to return when the
- * {@code expression} is {@code false}.
- * @return the $cond expression {@link BasicDBObject}.
- */
- public static BasicDBObject cond(final BasicDBObject expression, final Object thenResult, final Object elseResult) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) cond(builder, expression, thenResult, elseResult).get();
+ public static Document ifNull(final Object expression, final Object replacementExpression) {
+ return new Document("$ifNull", Arrays.asList(expression, replacementExpression));
}
/**
* Creates an "$cond" MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
* @param expression the expression {@link BasicDBObject}.
* @param thenResult the {@link Object} to return when the
* {@code expression} is {@code true}.
@@ -117,9 +70,7 @@ public final class ConditionalOperators {
* {@code expression} is {@code false}.
* @return the $cond expression {@link BasicDBObjectBuilder}.
*/
- public static BasicDBObjectBuilder cond(final BasicDBObjectBuilder builder, final BasicDBObject expression, final Object thenResult, final Object elseResult) {
- checkNotNull(builder);
- builder.add("$cond", ifThenElse(expression, thenResult, elseResult));
- return builder;
+ public static Document cond(final Document expression, final Object thenResult, final Object elseResult) {
+ return new Document("$cond", ifThenElse(expression, thenResult, elseResult));
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java
index 07dae18..a9f7595 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/document/operators/query/LogicalOperators.java
@@ -18,13 +18,13 @@
*/
package org.apache.rya.mongodb.document.operators.query;
-import static com.google.common.base.Preconditions.checkNotNull;
-
import java.util.List;
+import javax.xml.parsers.DocumentBuilder;
+
+import org.bson.Document;
+
import com.google.common.collect.Lists;
-import com.mongodb.BasicDBObject;
-import com.mongodb.BasicDBObjectBuilder;
/**
* Utility methods for logical operators.
@@ -41,62 +41,33 @@ public final class LogicalOperators {
* @param lhs the left-hand side operand.
* @param rhs the right-hand side operand.
* @param extras any additional operands.
- * @return the $and expression {@link BasicDBObject}.
+ * @return the $and expression {@link DocumentBuilder}.
*/
- public static BasicDBObject and(final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) and(builder, lhs, rhs, extras).get();
- }
-
- /**
- * Creates an $and MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
- * @param lhs the left-hand side operand.
- * @param rhs the right-hand side operand.
- * @param extras any additional operands.
- * @return the $and expression {@link BasicDBObjectBuilder}.
- */
- public static BasicDBObjectBuilder and(final BasicDBObjectBuilder builder, final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) {
- checkNotNull(builder);
- final List<BasicDBObject> operands = Lists.newArrayList(lhs, rhs);
+ public static Document and(final Document lhs, final Document rhs, final Document... extras) {
+ final List<Document> operands = Lists.newArrayList(lhs, rhs);
if (extras != null && extras.length > 0) {
operands.addAll(Lists.newArrayList(extras));
}
- builder.add("$and", operands);
- return builder;
- }
-
- /**
- * Creates an $or MongoDB expression.
- * @param lhs the left-hand side operand.
- * @param rhs the right-hand side operand.
- * @param extras any additional operands.
- * @return the $or expression {@link BasicDBObject}.
- */
- public static BasicDBObject or(final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) {
- final BasicDBObjectBuilder builder = BasicDBObjectBuilder.start();
- return (BasicDBObject) or(builder, lhs, rhs, extras).get();
+ return new Document("$and", operands);
}
/**
* Creates an $or MongoDB expression.
- * @param builder the {@link BasicDBObjectBuilder}. (not {@code null})
+ * @param builder the {@link DocumentBuilder}. (not {@code null})
* @param lhs the left-hand side operand.
* @param rhs the right-hand side operand.
* @param extras any additional operands.
- * @return the $or expression {@link BasicDBObjectBuilder}.
+ * @return the $or expression {@link DocumentBuilder}.
*/
- public static BasicDBObjectBuilder or(final BasicDBObjectBuilder builder, final BasicDBObject lhs, final BasicDBObject rhs, final BasicDBObject... extras) {
- checkNotNull(builder);
- final List<BasicDBObject> operands = Lists.newArrayList(lhs, rhs);
+ public static Document or(final Document lhs, final Document rhs, final Document... extras) {
+ final List<Document> operands = Lists.newArrayList(lhs, rhs);
if (extras != null && extras.length > 0) {
operands.addAll(Lists.newArrayList(extras));
}
- builder.add("$or", operands);
- return builder;
+ return new Document("$or", operands);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java
index 4807c4f..18f71d2 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementBindingSetCursorIterator.java
@@ -31,35 +31,37 @@ import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.persist.RyaDAOException;
import org.apache.rya.mongodb.dao.MongoDBStorageStrategy;
import org.apache.rya.mongodb.document.operators.aggregation.AggregationUtil;
+import org.bson.Document;
import org.openrdf.query.BindingSet;
import com.google.common.collect.Multimap;
-import com.mongodb.AggregationOutput;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DBCollection;
import com.mongodb.DBObject;
+import com.mongodb.client.AggregateIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
import info.aduna.iteration.CloseableIteration;
public class RyaStatementBindingSetCursorIterator implements CloseableIteration<Entry<RyaStatement, BindingSet>, RyaDAOException> {
private static final Logger log = Logger.getLogger(RyaStatementBindingSetCursorIterator.class);
- private final DBCollection coll;
+ private final MongoCollection<Document> coll;
private final Multimap<DBObject, BindingSet> rangeMap;
private final Iterator<DBObject> queryIterator;
private Long maxResults;
- private Iterator<DBObject> resultsIterator;
+ private Iterator<Document> resultsIterator;
private RyaStatement currentStatement;
private Collection<BindingSet> currentBindingSetCollection;
private Iterator<BindingSet> currentBindingSetIterator;
private final MongoDBStorageStrategy<RyaStatement> strategy;
private final Authorizations auths;
- public RyaStatementBindingSetCursorIterator(final DBCollection coll,
- final Multimap<DBObject, BindingSet> rangeMap, final MongoDBStorageStrategy<RyaStatement> strategy, final Authorizations auths) {
+ public RyaStatementBindingSetCursorIterator(final MongoCollection<Document> coll,
+ final Multimap<DBObject, BindingSet> rangeMap, final MongoDBStorageStrategy<RyaStatement> strategy,
+ final Authorizations auths) {
this.coll = coll;
this.rangeMap = rangeMap;
- this.queryIterator = rangeMap.keySet().iterator();
+ queryIterator = rangeMap.keySet().iterator();
this.strategy = strategy;
this.auths = auths;
}
@@ -94,8 +96,9 @@ public class RyaStatementBindingSetCursorIterator implements CloseableIteration<
}
if (currentResultCursorIsValid()) {
// convert to Rya Statement
- final DBObject queryResult = resultsIterator.next();
- currentStatement = strategy.deserializeDBObject(queryResult);
+ final Document queryResult = resultsIterator.next();
+ final DBObject dbo = (DBObject) JSON.parse(queryResult.toJson());
+ currentStatement = strategy.deserializeDBObject(dbo);
currentBindingSetIterator = currentBindingSetCollection.iterator();
}
}
@@ -106,12 +109,14 @@ public class RyaStatementBindingSetCursorIterator implements CloseableIteration<
currentBindingSetCollection = rangeMap.get(currentQuery);
// Executing redact aggregation to only return documents the user
// has access to.
- final List<DBObject> pipeline = new ArrayList<>();
- pipeline.add(new BasicDBObject("$match", currentQuery));
+ final List<Document> pipeline = new ArrayList<>();
+ pipeline.add(new Document("$match", currentQuery));
pipeline.addAll(AggregationUtil.createRedactPipeline(auths));
log.debug(pipeline);
- final AggregationOutput output = coll.aggregate(pipeline);
- resultsIterator = output.results().iterator();
+
+ final AggregateIterable<Document> aggIter = coll.aggregate(pipeline);
+ aggIter.batchSize(1000);
+ resultsIterator = aggIter.iterator();
if (resultsIterator.hasNext()) {
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-rya/blob/1d99212e/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java
----------------------------------------------------------------------
diff --git a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java
index 2f6fd44..1a5eb99 100644
--- a/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java
+++ b/dao/mongodb.rya/src/main/java/org/apache/rya/mongodb/iter/RyaStatementCursorIterator.java
@@ -29,27 +29,29 @@ import org.apache.rya.api.domain.RyaStatement;
import org.apache.rya.api.persist.RyaDAOException;
import org.apache.rya.mongodb.dao.MongoDBStorageStrategy;
import org.apache.rya.mongodb.document.operators.aggregation.AggregationUtil;
+import org.bson.Document;
-import com.mongodb.AggregationOutput;
-import com.mongodb.BasicDBObject;
-import com.mongodb.DBCollection;
import com.mongodb.DBObject;
+import com.mongodb.client.AggregateIterable;
+import com.mongodb.client.MongoCollection;
+import com.mongodb.util.JSON;
import info.aduna.iteration.CloseableIteration;
public class RyaStatementCursorIterator implements CloseableIteration<RyaStatement, RyaDAOException> {
private static final Logger log = Logger.getLogger(RyaStatementCursorIterator.class);
- private final DBCollection coll;
+ private final MongoCollection coll;
private final Iterator<DBObject> queryIterator;
- private Iterator<DBObject> resultsIterator;
+ private Iterator<Document> resultsIterator;
private final MongoDBStorageStrategy<RyaStatement> strategy;
private Long maxResults;
private final Authorizations auths;
- public RyaStatementCursorIterator(final DBCollection coll, final Set<DBObject> queries, final MongoDBStorageStrategy<RyaStatement> strategy, final Authorizations auths) {
- this.coll = coll;
- this.queryIterator = queries.iterator();
+ public RyaStatementCursorIterator(final MongoCollection<Document> collection, final Set<DBObject> queries,
+ final MongoDBStorageStrategy<RyaStatement> strategy, final Authorizations auths) {
+ coll = collection;
+ queryIterator = queries.iterator();
this.strategy = strategy;
this.auths = auths;
}
@@ -69,8 +71,9 @@ public class RyaStatementCursorIterator implements CloseableIteration<RyaStateme
}
if (currentCursorIsValid()) {
// convert to Rya Statement
- final DBObject queryResult = resultsIterator.next();
- final RyaStatement statement = strategy.deserializeDBObject(queryResult);
+ final Document queryResult = resultsIterator.next();
+ final DBObject dbo = (DBObject) JSON.parse(queryResult.toJson());
+ final RyaStatement statement = strategy.deserializeDBObject(dbo);
return statement;
}
return null;
@@ -82,12 +85,14 @@ public class RyaStatementCursorIterator implements CloseableIteration<RyaStateme
// Executing redact aggregation to only return documents the user
// has access to.
- final List<DBObject> pipeline = new ArrayList<>();
- pipeline.add(new BasicDBObject("$match", currentQuery));
+ final List<Document> pipeline = new ArrayList<>();
+ pipeline.add(new Document("$match", currentQuery));
pipeline.addAll(AggregationUtil.createRedactPipeline(auths));
log.debug(pipeline);
- final AggregationOutput output = coll.aggregate(pipeline);
- resultsIterator = output.results().iterator();
+ final AggregateIterable<Document> output = coll.aggregate(pipeline);
+ output.batchSize(1000);
+
+ resultsIterator = output.iterator();
if (resultsIterator.hasNext()) {
break;
}