You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@calcite.apache.org by jh...@apache.org on 2014/08/30 02:17:51 UTC
[1/2] git commit: [OPTIQ-269] MongoDB result sets larger than 16MB
Repository: incubator-optiq
Updated Branches:
refs/heads/master d30a76bb1 -> e05f70c75
[OPTIQ-269] MongoDB result sets larger than 16MB
Detects the version of mongod to determine which kind of intermediate
result to utilize.
For versions 2.6+, a Cursor result will be returned form the
DBCollection otherwise a AggregationOutput result is returned.
An Iterator<DBObject> object is retrieved from the intermediate result
and returned by MongoTable.aggregate.
Updated dependency in for mongo-java-driver in master/mongodb/pom.xml:
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
<version>2.12.3</version>
</dependency>
Close apache/incubator-optiq#6
Project: http://git-wip-us.apache.org/repos/asf/incubator-optiq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-optiq/commit/97acf7db
Tree: http://git-wip-us.apache.org/repos/asf/incubator-optiq/tree/97acf7db
Diff: http://git-wip-us.apache.org/repos/asf/incubator-optiq/diff/97acf7db
Branch: refs/heads/master
Commit: 97acf7db157d9902db94c7b1e993890260805804
Parents: d30a76b
Author: Joe Centenni <jc...@panix.com>
Authored: Thu Jul 31 20:11:14 2014 -0400
Committer: Julian Hyde <jh...@apache.org>
Committed: Fri Aug 29 16:07:45 2014 -0700
----------------------------------------------------------------------
.../optiq/impl/mongodb/MongoTable.java | 69 ++++++++++++++++++--
pom.xml | 2 +-
2 files changed, 65 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-optiq/blob/97acf7db/mongodb/src/main/java/net/hydromatic/optiq/impl/mongodb/MongoTable.java
----------------------------------------------------------------------
diff --git a/mongodb/src/main/java/net/hydromatic/optiq/impl/mongodb/MongoTable.java b/mongodb/src/main/java/net/hydromatic/optiq/impl/mongodb/MongoTable.java
index f5a0c40..3741a23 100644
--- a/mongodb/src/main/java/net/hydromatic/optiq/impl/mongodb/MongoTable.java
+++ b/mongodb/src/main/java/net/hydromatic/optiq/impl/mongodb/MongoTable.java
@@ -118,7 +118,18 @@ public class MongoTable extends AbstractQueryableTable
public Enumerable<Object> aggregate(final DB mongoDb,
final List<Map.Entry<String, Class>> fields,
final List<String> operations) {
- List<DBObject> list = new ArrayList<DBObject>();
+ final List<DBObject> list = new ArrayList<DBObject>();
+ final BasicDBList versionArray = (BasicDBList) mongoDb
+ .command("buildInfo").get("versionArray");
+ final Integer versionMajor = parseIntString(versionArray
+ .get(0).toString());
+ final Integer versionMinor = parseIntString(versionArray
+ .get(1).toString());
+// final Integer versionMaintenance = parseIntString(versionArray
+// .get(2).toString());
+// final Integer versionBuild = parseIntString(versionArray
+// .get(3).toString());
+
for (String operation : operations) {
list.add((DBObject) JSON.parse(operation));
}
@@ -128,19 +139,67 @@ public class MongoTable extends AbstractQueryableTable
MongoEnumerator.getter(fields);
return new AbstractEnumerable<Object>() {
public Enumerator<Object> enumerator() {
- final AggregationOutput result;
+ final Iterator<DBObject> resultIterator;
try {
- result = mongoDb.getCollection(collectionName)
- .aggregate(first, rest.toArray(new DBObject[rest.size()]));
+ // Changed in version 2.6: The db.collection.aggregate() method
+ // returns a cursor
+ // and can return result sets of any size.
+ // See: http://docs.mongodb.org/manual/core/aggregation-pipeline
+ if (versionMajor > 1) {
+ // MongoDB version 2.6+
+ if (versionMinor > 5) {
+ AggregationOptions options = AggregationOptions.builder()
+ .outputMode(AggregationOptions.OutputMode.CURSOR).build();
+ // Warning - this can result in a very large ArrayList!
+ // but you should know your data and aggregate accordingly
+ ArrayList<DBObject> resultAsArrayList
+ = new ArrayList<DBObject>(Util.toList(mongoDb.
+ getCollection(collectionName)
+ .aggregate(list, options)));
+ resultIterator = resultAsArrayList.iterator();
+ } else { // Pre MongoDB version 2.6
+ AggregationOutput result = aggregateOldWay(mongoDb
+ .getCollection(collectionName), first, rest);
+ resultIterator = result.results().iterator();
+ }
+ } else { // Pre MongoDB version 2
+ AggregationOutput result = aggregateOldWay(mongoDb
+ .getCollection(collectionName), first, rest);
+ resultIterator = result.results().iterator();
+ }
} catch (Exception e) {
throw new RuntimeException("While running MongoDB query "
+ Util.toString(operations, "[", ",\n", "]"), e);
}
- return new MongoEnumerator(result.results().iterator(), getter);
+ return new MongoEnumerator(resultIterator, getter);
}
};
}
+ /** Helper method to strip non-numerics from a string
+ * <p>Currently used to determine mongod versioning numbers
+ * from buildInfo.versionArray for use in aggregate method logic</p>
+ * @param valueString
+ * @return Integer */
+ private static Integer parseIntString(String valueString) {
+ return Integer.parseInt(valueString.replaceAll("[^0-9]", ""));
+ }
+
+ /** Executes an "aggregate" operation for pre-2.6 mongo servers.
+ * <p>Return document is limited to 4M or 16M in size depending on
+ * version of mongo <p>Helper method for
+ * {@link net.hydromatic.optiq.impl.mongodb.MongoTable#aggregate}
+ * </p>
+ * @param dbCollection
+ * @param first the first aggregate action
+ * @param rest the rest of the aggregate actions
+ * @return AggregationOutput */
+ private AggregationOutput aggregateOldWay(DBCollection dbCollection,
+ DBObject first, List<DBObject> rest) {
+ return dbCollection.aggregate(first, rest
+ .toArray(new DBObject[rest.size()]));
+ }
+
/** Implementation of {@link net.hydromatic.linq4j.Queryable} based on
* a {@link net.hydromatic.optiq.impl.mongodb.MongoTable}. */
public static class MongoQueryable<T> extends AbstractTableQueryable<T> {
http://git-wip-us.apache.org/repos/asf/incubator-optiq/blob/97acf7db/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index cd2f019..d89621e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -206,7 +206,7 @@ limitations under the License.
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
- <version>2.11.1</version>
+ <version>2.12.3</version>
</dependency>
<dependency>
<groupId>org.openjdk.jmh</groupId>
[2/2] git commit: Removed hardcoded foodmart schema information
Posted by jh...@apache.org.
Removed hardcoded foodmart schema information
Project: http://git-wip-us.apache.org/repos/asf/incubator-optiq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-optiq/commit/e05f70c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-optiq/tree/e05f70c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-optiq/diff/e05f70c7
Branch: refs/heads/master
Commit: e05f70c7572e8d28af4e861eaca6b9a0e62a11e7
Parents: 97acf7d
Author: Marc Prud'hommeaux <mw...@cornell.edu>
Authored: Sun Jul 6 13:38:25 2014 -0400
Committer: Julian Hyde <jh...@apache.org>
Committed: Fri Aug 29 16:14:14 2014 -0700
----------------------------------------------------------------------
.../java/net/hydromatic/optiq/model/ModelHandler.java | 13 -------------
1 file changed, 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-optiq/blob/e05f70c7/core/src/main/java/net/hydromatic/optiq/model/ModelHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/net/hydromatic/optiq/model/ModelHandler.java b/core/src/main/java/net/hydromatic/optiq/model/ModelHandler.java
index 91cd1bf..7c47d73 100644
--- a/core/src/main/java/net/hydromatic/optiq/model/ModelHandler.java
+++ b/core/src/main/java/net/hydromatic/optiq/model/ModelHandler.java
@@ -139,19 +139,6 @@ public class ModelHandler {
schema.setPath(stringListList(jsonSchema.path));
}
populateSchema(jsonSchema, schema);
- if (schema.getName().equals("mat")) {
- // Inject by hand a Star Table. Later we'll add a JSON model element.
- final List<Table> tables = new ArrayList<Table>();
- final String[] tableNames = {
- "sales_fact_1997", "time_by_day", "product", "product_class"
- };
- final SchemaPlus schema2 = parentSchema.getSubSchema("foodmart");
- for (String tableName : tableNames) {
- tables.add(schema2.getTable(tableName));
- }
- final String tableName = "star";
- schema.add(tableName, StarTable.of(tables));
- }
}
private static ImmutableList<ImmutableList<String>> stringListList(