You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by dz...@apache.org on 2022/01/30 11:26:47 UTC
[drill] branch master updated: DRILL-8118: Add Option to Allow Disk Use on Mongo Queries (#2442)
This is an automated email from the ASF dual-hosted git repository.
dzamo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 30104d9 DRILL-8118: Add Option to Allow Disk Use on Mongo Queries (#2442)
30104d9 is described below
commit 30104d9faf3cf9fe3fcae4d109dc28f20f27f915
Author: Charles S. Givre <cg...@apache.org>
AuthorDate: Sun Jan 30 06:26:39 2022 -0500
DRILL-8118: Add Option to Allow Disk Use on Mongo Queries (#2442)
* Initial commit
* Update contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
Co-authored-by: luoc <lu...@apache.org>
Co-authored-by: luoc <lu...@apache.org>
---
.../java/org/apache/drill/exec/store/mongo/MongoRecordReader.java | 6 +++++-
.../apache/drill/exec/store/mongo/MongoStoragePluginConfig.java | 8 ++++++++
.../src/main/resources/bootstrap-storage-plugins.json | 1 +
.../java/org/apache/drill/exec/store/mongo/MongoTestBase.java | 2 +-
.../store/mongo/TestMongoStoragePluginUsesCredentialsStore.java | 2 +-
5 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
index 08219f6..93b2d16 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoRecordReader.java
@@ -198,7 +198,11 @@ public class MongoRecordReader extends AbstractRecordReader {
if (!fields.isEmpty()) {
operations.add(Aggregates.project(fields));
}
- projection = collection.aggregate(operations);
+ if (plugin.getConfig().allowDiskUse()) {
+ projection = collection.aggregate(operations).allowDiskUse(true);
+ } else {
+ projection = collection.aggregate(operations);
+ }
} else {
projection = collection.find(filters).projection(fields);
}
diff --git a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
index be410ae..e1da24b 100644
--- a/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
+++ b/contrib/storage-mongo/src/main/java/org/apache/drill/exec/store/mongo/MongoStoragePluginConfig.java
@@ -40,6 +40,8 @@ public class MongoStoragePluginConfig extends AbstractSecuredStoragePluginConfig
private final String connection;
+ private final boolean allowDiskUse;
+
@JsonIgnore
private final ConnectionString clientURI;
@@ -51,12 +53,14 @@ public class MongoStoragePluginConfig extends AbstractSecuredStoragePluginConfig
public MongoStoragePluginConfig(@JsonProperty("connection") String connection,
@JsonProperty("pluginOptimizations") MongoPluginOptimizations pluginOptimizations,
@JsonProperty("batchSize") Integer batchSize,
+ @JsonProperty("allowDiskUse") Boolean allowDiskUse,
@JsonProperty("credentialsProvider") CredentialsProvider credentialsProvider) {
super(getCredentialsProvider(credentialsProvider), credentialsProvider == null);
this.connection = connection;
this.clientURI = new ConnectionString(connection);
this.pluginOptimizations = ObjectUtils.defaultIfNull(pluginOptimizations, new MongoPluginOptimizations());
this.batchSize = batchSize != null ? batchSize : 100;
+ this.allowDiskUse = allowDiskUse != null && allowDiskUse;
}
public MongoPluginOptimizations getPluginOptimizations() {
@@ -76,6 +80,10 @@ public class MongoStoragePluginConfig extends AbstractSecuredStoragePluginConfig
return batchSize;
}
+ public boolean allowDiskUse() {
+ return allowDiskUse;
+ }
+
private static CredentialsProvider getCredentialsProvider(CredentialsProvider credentialsProvider) {
return credentialsProvider != null ? credentialsProvider : PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER;
}
diff --git a/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json b/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
index 2010e32..7e3cd42 100644
--- a/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
+++ b/contrib/storage-mongo/src/main/resources/bootstrap-storage-plugins.json
@@ -3,6 +3,7 @@
"mongo" : {
"type":"mongo",
"connection":"mongodb://localhost:27017/",
+ "allowDiskUse": false,
"enabled": false
}
}
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
index 8224470..4b3cc1a 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
@@ -39,7 +39,7 @@ public class MongoTestBase extends ClusterTest implements MongoTestConstants {
private static void initMongoStoragePlugin(String connectionURI) throws Exception {
MongoStoragePluginConfig storagePluginConfig = new MongoStoragePluginConfig(connectionURI,
- null, 100, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
+ null, 100, false, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER);
storagePluginConfig.setEnabled(true);
pluginRegistry.put(MongoStoragePluginConfig.NAME, storagePluginConfig);
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
index 298744b..7a09abe 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestMongoStoragePluginUsesCredentialsStore.java
@@ -35,7 +35,7 @@ public class TestMongoStoragePluginUsesCredentialsStore extends BaseTest {
private void test(String expectedUserName, String expectedPassword, String connection, String name) throws ExecutionSetupException {
MongoStoragePlugin plugin = new MongoStoragePlugin(
- new MongoStoragePluginConfig(connection, null, 100, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER),
+ new MongoStoragePluginConfig(connection, null, 100, false, PlainCredentialsProvider.EMPTY_CREDENTIALS_PROVIDER),
null, name);
MongoClientImpl client = (MongoClientImpl) plugin.getClient();
MongoCredential cred = client.getSettings().getCredential();