You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by su...@apache.org on 2017/02/25 07:18:01 UTC
[08/29] drill git commit: DRILL-5196: Init MongoDB cluster when run a
single test case directly through command line or IDE
DRILL-5196: Init MongoDB cluster when run a single test case directly through command line or IDE
Other fixes include:
+ Sync mongo-java-driver versions to newer 3.2.0
+ update flapdoodle package to latest accordingly
closes #741
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/cd8af10f
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/cd8af10f
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/cd8af10f
Branch: refs/heads/master
Commit: cd8af10f70a6bab4795e24b882d277ade06d402b
Parents: b892b99
Author: chunhui-shi <cs...@maprtech.com>
Authored: Fri Jan 13 17:20:46 2017 -0800
Committer: Sudheesh Katkam <su...@apache.org>
Committed: Fri Feb 24 19:01:40 2017 -0800
----------------------------------------------------------------------
contrib/storage-mongo/pom.xml | 2 +-
.../drill/exec/store/mongo/MongoTestBase.java | 2 +
.../exec/store/mongo/MongoTestConstants.java | 4 ++
.../drill/exec/store/mongo/MongoTestSuit.java | 59 +++++++++++++-------
.../exec/store/mongo/TestTableGenerator.java | 13 ++++-
.../src/test/resources/datatype-oid.json | 1 +
exec/java-exec/pom.xml | 2 +-
7 files changed, 58 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/contrib/storage-mongo/pom.xml
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/pom.xml b/contrib/storage-mongo/pom.xml
index 8ce34ef..c7c2b0d 100644
--- a/contrib/storage-mongo/pom.xml
+++ b/contrib/storage-mongo/pom.xml
@@ -69,7 +69,7 @@
<dependency>
<groupId>de.flapdoodle.embed</groupId>
<artifactId>de.flapdoodle.embed.mongo</artifactId>
- <version>1.50.0</version>
+ <version>1.50.5</version>
<scope>test</scope>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
index 1877a64..ec97514 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestBase.java
@@ -36,6 +36,7 @@ public class MongoTestBase extends PlanTestBase implements MongoTestConstants {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
+ MongoTestSuit.initMongo();
initMongoStoragePlugin();
}
@@ -84,6 +85,7 @@ public class MongoTestBase extends PlanTestBase implements MongoTestConstants {
@AfterClass
public static void tearDownMongoTestBase() throws Exception {
+ MongoTestSuit.tearDownCluster();
storagePlugin = null;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
index 0ff031a..290c3c3 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestConstants.java
@@ -44,6 +44,10 @@ public interface MongoTestConstants {
public static final String EMP_DATA = "emp.json";
public static final String SCHEMA_CHANGE_DATA = "schema_change_int_to_string.json";
+ String DATATYPE_DB = "datatype";
+ String DATATYPE_COLLECTION = "types";
+ String DATATYPE_DATA = "datatype-oid.json";
+
public static final String REPLICA_SET_1_NAME = "shard_1_replicas";
public static final String REPLICA_SET_2_NAME = "shard_2_replicas";
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
index 01ee915..c34afe3 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/MongoTestSuit.java
@@ -23,6 +23,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
import org.bson.Document;
import org.bson.conversions.Bson;
@@ -73,6 +74,8 @@ public class MongoTestSuit implements MongoTestConstants {
private static boolean authEnabled = System.getProperty(
"drill.mongo.tests.authEnabled", "false").equalsIgnoreCase("true");
+ private static volatile AtomicInteger initCount = new AtomicInteger(0);
+
private static class DistributedMode {
private static MongosSystemForTestFactory mongosTestFactory;
@@ -116,11 +119,12 @@ public class MongoTestSuit implements MongoTestConstants {
}
createDbAndCollections(DONUTS_DB, DONUTS_COLLECTION, "id");
createDbAndCollections(EMPLOYEE_DB, EMPTY_COLLECTION, "field_2");
+ createDbAndCollections(DATATYPE_DB, DATATYPE_COLLECTION, "_id");
}
private static IMongodConfig crateConfigServerConfig(int configServerPort,
boolean flag) throws UnknownHostException, IOException {
- IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().verbose(false)
+ IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
.build();
IMongodConfig mongodConfig = new MongodConfigBuilder()
@@ -133,7 +137,7 @@ public class MongoTestSuit implements MongoTestConstants {
private static IMongodConfig crateIMongodConfig(int mongodPort,
boolean flag, String replicaName) throws UnknownHostException,
IOException {
- IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().verbose(false)
+ IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
.build();
Storage replication = new Storage(null, replicaName, 0);
@@ -147,7 +151,7 @@ public class MongoTestSuit implements MongoTestConstants {
private static IMongosConfig createIMongosConfig()
throws UnknownHostException, IOException {
- IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().verbose(false)
+ IMongoCmdOptions cmdOptions = new MongoCmdOptionsBuilder().useNoJournal(false).verbose(false)
.build();
IMongosConfig mongosConfig = new MongosConfigBuilder()
@@ -189,6 +193,7 @@ public class MongoTestSuit implements MongoTestConstants {
createDbAndCollections(EMPLOYEE_DB, EMPINFO_COLLECTION, "employee_id");
createDbAndCollections(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, "field_2");
createDbAndCollections(EMPLOYEE_DB, EMPTY_COLLECTION, "field_2");
+ createDbAndCollections(DATATYPE_DB, DATATYPE_COLLECTION, "_id");
}
private static void cleanup() {
@@ -204,16 +209,20 @@ public class MongoTestSuit implements MongoTestConstants {
@BeforeClass
public static void initMongo() throws Exception {
synchronized (MongoTestSuit.class) {
- if (distMode) {
- logger.info("Executing tests in distributed mode");
- DistributedMode.setup();
- } else {
- logger.info("Executing tests in single mode");
- SingleMode.setup();
+ if (initCount.get() == 0) {
+ if (distMode) {
+ logger.info("Executing tests in distributed mode");
+ DistributedMode.setup();
+ } else {
+ logger.info("Executing tests in single mode");
+ SingleMode.setup();
+ }
+ TestTableGenerator.importData(EMPLOYEE_DB, EMPINFO_COLLECTION, EMP_DATA);
+ TestTableGenerator.importData(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, SCHEMA_CHANGE_DATA);
+ TestTableGenerator.importData(DONUTS_DB, DONUTS_COLLECTION, DONUTS_DATA);
+ TestTableGenerator.importData(DATATYPE_DB, DATATYPE_COLLECTION, DATATYPE_DATA);
}
- TestTableGenerator.importData(EMPLOYEE_DB, EMPINFO_COLLECTION, EMP_DATA);
- TestTableGenerator.importData(EMPLOYEE_DB, SCHEMA_CHANGE_COLLECTION, SCHEMA_CHANGE_DATA);
- TestTableGenerator.importData(DONUTS_DB, DONUTS_COLLECTION, DONUTS_DATA);
+ initCount.incrementAndGet();
}
}
@@ -234,15 +243,25 @@ public class MongoTestSuit implements MongoTestConstants {
@AfterClass
public static void tearDownCluster() throws Exception {
- if (mongoClient != null) {
- mongoClient.dropDatabase(EMPLOYEE_DB);
- mongoClient.close();
- }
synchronized (MongoTestSuit.class) {
- if (distMode) {
- DistributedMode.cleanup();
- } else {
- SingleMode.cleanup();
+ if (initCount.decrementAndGet() == 0) {
+ try {
+ if (mongoClient != null) {
+ mongoClient.dropDatabase(EMPLOYEE_DB);
+ mongoClient.dropDatabase(DATATYPE_DB);
+ mongoClient.dropDatabase(DONUTS_DB);
+ }
+ }
+ finally {
+ if (mongoClient != null) {
+ mongoClient.close();
+ }
+ if (distMode) {
+ DistributedMode.cleanup();
+ } else {
+ SingleMode.cleanup();
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
index 62e1204..ffe3791 100644
--- a/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
+++ b/contrib/storage-mongo/src/test/java/org/apache/drill/exec/store/mongo/TestTableGenerator.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.store.mongo;
import java.io.IOException;
import java.net.UnknownHostException;
+import de.flapdoodle.embed.mongo.MongoImportProcess;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,7 +41,7 @@ public class TestTableGenerator implements MongoTestConstants {
.getLogger(TestTableGenerator.class);
public static void importData(String dbName, String collectionName,
- String fileName) throws IOException {
+ String fileName) throws InterruptedException,IOException {
String jsonFile = Resources.getResource(fileName).toString();
jsonFile = jsonFile.replaceFirst("file:", StringUtils.EMPTY);
generateTable(dbName, collectionName, jsonFile, true, true, false);
@@ -48,7 +49,7 @@ public class TestTableGenerator implements MongoTestConstants {
public static void generateTable(String dbName, String collection,
String jsonFile, Boolean jsonArray, Boolean upsert, Boolean drop)
- throws UnknownHostException, IOException {
+ throws InterruptedException, IOException {
logger.info("Started importing file {} into collection {} ", jsonFile,
collection);
IMongoImportConfig mongoImportConfig = new MongoImportConfigBuilder()
@@ -58,7 +59,13 @@ public class TestTableGenerator implements MongoTestConstants {
.jsonArray(jsonArray).importFile(jsonFile).build();
MongoImportExecutable importExecutable = MongoImportStarter
.getDefaultInstance().prepare(mongoImportConfig);
- importExecutable.start();
+ MongoImportProcess importProcess = importExecutable.start();
+
+ // import is in a separate process, we should wait until the process exit
+ while (importProcess.isProcessRunning()) {
+ Thread.sleep(1000);
+ }
+
logger.info("Imported file {} into collection {} ", jsonFile, collection);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/contrib/storage-mongo/src/test/resources/datatype-oid.json
----------------------------------------------------------------------
diff --git a/contrib/storage-mongo/src/test/resources/datatype-oid.json b/contrib/storage-mongo/src/test/resources/datatype-oid.json
new file mode 100644
index 0000000..52058e8
--- /dev/null
+++ b/contrib/storage-mongo/src/test/resources/datatype-oid.json
@@ -0,0 +1 @@
+[{"_id": {"$oid" : "582081d96b69060001fd8938"}, "account" : { "$ref": "contact", "$id": {"$oid": "999cbf116b69060001fd8611"} }}, {"_id" : {"$oid" : "582081d96b69060001fd8939" } } ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/cd8af10f/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/exec/java-exec/pom.xml b/exec/java-exec/pom.xml
index 2acda3a..24d3c93 100644
--- a/exec/java-exec/pom.xml
+++ b/exec/java-exec/pom.xml
@@ -156,7 +156,7 @@
<dependency>
<groupId>org.mongodb</groupId>
<artifactId>mongo-java-driver</artifactId>
- <version>3.0.2</version>
+ <version>3.2.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>