You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by dr...@apache.org on 2020/03/21 22:39:48 UTC

[gora] 07/07: GORA-647 Refactor GoraMongodbAuthenticationTestDriver

This is an automated email from the ASF dual-hosted git repository.

drazzib pushed a commit to branch GORA-647-test-containers-mongo
in repository https://gitbox.apache.org/repos/asf/gora.git

commit 1d7b76548e085a6ad104c7eff646c7fdde7c1a3c
Author: Damien Raude-Morvan <dr...@drazzib.com>
AuthorDate: Sat Mar 21 23:39:15 2020 +0100

    GORA-647 Refactor GoraMongodbAuthenticationTestDriver
    
    * Use MongoDB docker container feature to initialize superuser (via env)
    * Pass --auth and --setParameter authenticationMechanisms as commands
    * Use mongo shell CLI - inside container - to execute script
---
 .../GoraMongodbAuthenticationTestDriver.java       | 194 ++++++---------------
 .../authentications/PLAIN_AuthenticationTest.java  |   6 +-
 .../SCRAM_SHA_1_AuthenticationTest.java            |   9 +-
 3 files changed, 56 insertions(+), 153 deletions(-)

diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java
index 38e8616..b96fa7f 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/GoraMongodbAuthenticationTestDriver.java
@@ -17,91 +17,55 @@
  */
 package org.apache.gora.mongodb.authentications;
 
-import de.flapdoodle.embed.mongo.*;
-import de.flapdoodle.embed.mongo.config.*;
-import de.flapdoodle.embed.mongo.distribution.Version;
-import de.flapdoodle.embed.process.config.IRuntimeConfig;
-import de.flapdoodle.embed.process.config.io.ProcessOutput;
-import de.flapdoodle.embed.process.io.IStreamProcessor;
-import de.flapdoodle.embed.process.io.LogWatchStreamProcessor;
-import de.flapdoodle.embed.process.io.NamedOutputStreamProcessor;
-import de.flapdoodle.embed.process.runtime.Network;
+import com.mongodb.ServerAddress;
 import org.apache.gora.GoraTestDriver;
+import org.apache.gora.mongodb.MongoContainer;
 import org.apache.gora.mongodb.store.MongoStore;
 import org.apache.gora.mongodb.store.MongoStoreParameters;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.Container;
 
-import java.io.File;
 import java.io.IOException;
-import java.io.PrintWriter;
-import java.util.Collections;
-import java.util.HashSet;
+import java.time.Duration;
 
-import static de.flapdoodle.embed.process.io.Processors.console;
-import static de.flapdoodle.embed.process.io.Processors.namedConsole;
-import static java.util.Arrays.asList;
 import static org.apache.commons.lang3.StringUtils.isEmpty;
 
 /**
  * Driver to set up an embedded MongoDB database instance for use in our  * unit tests.
  * This class is specially written to automate authentication mechanisms.
- * We use embedded mongodb which is available from
- * https://github.com/flapdoodle-oss/embedmongo.flapdoodle.de
  */
 class GoraMongodbAuthenticationTestDriver extends GoraTestDriver {
     private static final Logger log = LoggerFactory.getLogger(GoraMongodbAuthenticationTestDriver.class);
-    private static final int INIT_TIMEOUT_MS = 30000;
-    private static final String USER_ADDED_TOKEN = "Successfully added user";
-    private ThreadLocal<Boolean> started = new ThreadLocal<>();
-    private int port;
-    private MongodExecutable _mongodExe;
-    private MongodProcess _mongod;
-    private MongodStarter runtime;
-    private IMongodConfig mongodConfig;
-    private String adminUsername = "madhawa";
-    private String adminPassword = "123";
-    private Version.Main useVersion;
-    private String authMechanisms;
-    private boolean auth = false;
-
-    GoraMongodbAuthenticationTestDriver(String authMechanisms, Version.Main useVersion) throws IOException {
+    private MongoContainer _container;
+    private final String adminUsername = "madhawa";
+    private final String adminPassword = "123";
+    private final String useVersion;
+    private final String authMechanisms;
+
+    GoraMongodbAuthenticationTestDriver(String authMechanisms, String useVersion) {
         super(MongoStore.class);
         this.authMechanisms = authMechanisms;
         this.useVersion = useVersion;
-        started.set(false);
-        if (!this.authMechanisms.equals("MONGODB-CR")) {
-            auth = true;
-        }
-
     }
 
     private void doStart() throws Exception {
-        IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder()
-                .defaultsWithLogger(Command.MongoD, log)
-                .processOutput(ProcessOutput.getDefaultInstanceSilent())
-                .build();
-        runtime = MongodStarter.getInstance(runtimeConfig);
         try {
-            log.info("Starting the mongo server without authentications");
+            log.info("Starting the embedded Mongodb server");
             startWithAuth();
-            log.info("Adding admin user");
-            addAdmin();
-            if (this.authMechanisms.equals("SCRAM-SHA-1")) {
+            if (authMechanisms.equals("SCRAM-SHA-1")) {
                 setSCRAM_SHA_1Credentials();
             }
-            if (this.authMechanisms.equals("MONGODB-CR")) {
-                setMongoDB_CRCredentials();
-                tearDownClass();
-                auth = true;
-                startWithAuth();
-                addAdmin();
-            }
             // Store Mongo server "host:port" in Hadoop configuration
             // so that MongoStore will be able to get it latter
-            conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, "127.0.0.1:" + port);
+            ServerAddress address = _container.getServerAddress();
+            int port = address.getPort();
+            String host = address.getHost();
+            String mongoServersProp = String.format("%s:%d", host, port);
+
+            conf.set(MongoStoreParameters.PROP_MONGO_SERVERS, mongoServersProp);
             conf.set(MongoStoreParameters.PROP_MONGO_DB, "admin");
-            conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, this.authMechanisms);
+            conf.set(MongoStoreParameters.PROP_MONGO_AUTHENTICATION_TYPE, authMechanisms);
             conf.set(MongoStoreParameters.PROP_MONGO_LOGIN, adminUsername);
             conf.set(MongoStoreParameters.PROP_MONGO_SECRET, adminPassword);
         } catch (Exception e) {
@@ -112,11 +76,8 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver {
 
     private void startWithAuth() throws IOException {
         try {
-            if(!started.get()) {
-                prepareExecutable();
-                _mongod = _mongodExe.start();
-                started.set(true);
-            }
+            prepareExecutable();
+            _container.start();
         } catch (Exception e) {
             log.error("Error starting embedded Mongodb server... tearing down test driver.");
             tearDownClass();
@@ -124,96 +85,51 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver {
     }
 
     private void prepareExecutable() throws IOException {
-        final MongoCmdOptionsBuilder cmdBuilder = new MongoCmdOptionsBuilder();
-        cmdBuilder.enableAuth(auth);
-        final IMongoCmdOptions cmdOptions = cmdBuilder.build();
-        MongodConfigBuilder builder = new MongodConfigBuilder()
-                .version(useVersion)
-                .cmdOptions(cmdOptions)
-                .net(new Net(port, Network.localhostIsIPv6()));
-        if (auth) {
-            builder.setParameter("authenticationMechanisms", authMechanisms);
-        }
-
-        mongodConfig = builder.build();
-        _mongodExe = runtime.prepare(mongodConfig);
-    }
-
-    private void addAdmin() throws IOException, InterruptedException {
-        final String scriptText = "db.createUser(\n" +
-                "  {\n" +
-                "    user: \"madhawa\",\n" +
-                "    pwd: \"123\",\n" +
-                "    roles: [ { role: \"root\", db: \"admin\" } ]\n" +
-                "  }\n" +
-                ");";
-        runScriptAndWait(scriptText, USER_ADDED_TOKEN, new String[]{"couldn't add user", "failed to load", "login failed"}, "admin", null, null);
-    }
-
-    private void setMongoDB_CRCredentials() throws Exception {
-        final String scriptText1 = "var schema = db.system.version.findOne({\"_id\" : \"authSchema\"});\nschema.currentVersion = 3;\ndb.system.version.save(schema);\n";
-      //  final String scriptText1 = "db.system.version.remove({});\ndb.system.version.insert({ \"_id\" : \"authSchema\", \"currentVersion\" : 3 });";
-        runScriptAndWait(scriptText1, "Successfully added authSchema", null, "admin", adminUsername, adminPassword);
+        _container = new MongoContainer(useVersion);
+        // https://hub.docker.com/_/mongo
+        // These variables, used in conjunction, create a new user and set that user's password.
+        // This user is created in the admin authentication database
+        // and given the role of root, which is a "superuser" role.
+        _container.withEnv("MONGO_INITDB_ROOT_USERNAME", adminUsername);
+        _container.withEnv("MONGO_INITDB_ROOT_PASSWORD", adminPassword);
+
+        // To enable authentication, MongoDB will have to restart itself
+        // so wait for at least 5 sec
+        _container.withMinimumRunningDuration(Duration.ofSeconds(5));
+
+        // https://docs.mongodb.com/manual/tutorial/enable-authentication/
+        // https://docs.mongodb.com/manual/reference/parameters/#param.authenticationMechanisms
+        _container.withCommand("--auth", "--setParameter", "authenticationMechanisms=" + authMechanisms);
     }
 
     private void setSCRAM_SHA_1Credentials() throws Exception {
         final String scriptText1 = "db.adminCommand({authSchemaUpgrade: 1});\n";
-        runScriptAndWait(scriptText1, "Successfully added authSchema", null, "admin", adminUsername, adminPassword);
+        runScriptAndWait(scriptText1, "admin", adminUsername, adminPassword);
     }
 
-    private void runScriptAndWait(String scriptText, String token, String[] failures, String dbName, String username, String password) throws IOException {
-        IStreamProcessor mongoOutput;
-        if (!isEmpty(token)) {
-            mongoOutput = new LogWatchStreamProcessor(
-                    token,
-                    (failures != null) ? new HashSet<>(asList(failures)) : Collections.emptySet(),
-                    namedConsole("[mongo shell output]"));
-        } else {
-            mongoOutput = new NamedOutputStreamProcessor("[mongo shell output]", console());
-        }
-        IRuntimeConfig runtimeConfig = new RuntimeConfigBuilder()
-                .defaults(Command.Mongo)
-                .processOutput(new ProcessOutput(
-                        mongoOutput,
-                        namedConsole("[mongo shell error]"),
-                        console()))
-                .build();
-        MongoShellStarter starter = MongoShellStarter.getInstance(runtimeConfig);
-
-        final File scriptFile = writeTmpScriptFile(scriptText);
-        final MongoShellConfigBuilder builder = new MongoShellConfigBuilder();
-        if (!isEmpty(dbName)) {
-            builder.dbName(dbName);
-        }
+    private void runScriptAndWait(String scriptText, String dbName, String username, String password)
+            throws InterruptedException, IOException {
+        final StringBuilder builder = new StringBuilder("mongo --quiet");
         if (!isEmpty(username)) {
-            builder.username(username);
+            builder.append(" --username ").append(username);
         }
         if (!isEmpty(password)) {
-            builder.password(password);
+            builder.append(" --password ").append(password);
         }
-        starter.prepare(builder
-                .scriptName(scriptFile.getAbsolutePath())
-                .version(mongodConfig.version())
-                .net(mongodConfig.net())
-                .build()).start();
-        if (mongoOutput instanceof LogWatchStreamProcessor) {
-            ((LogWatchStreamProcessor) mongoOutput).waitForResult(INIT_TIMEOUT_MS);
+        if (!isEmpty(dbName)) {
+            builder.append(" ").append(dbName);
         }
-    }
+        builder.append(" --eval '").append(scriptText).append("'");
 
-    private File writeTmpScriptFile(String scriptText) throws IOException {
-        File scriptFile = File.createTempFile("tempfile", ".js");
-        scriptFile.deleteOnExit();
-        PrintWriter writer = new PrintWriter(scriptFile, "UTF-8");
-        writer.write(scriptText);
-        writer.close();
-        return scriptFile;
+        Container.ExecResult res = _container.execInContainer("/bin/bash", "-c", builder.toString());
+        if (!isEmpty(res.getStderr())) {
+            log.error("Unable to run script on Mongodb server {}: {}", scriptText, res.getStderr());
+            throw new IOException(res.getStderr());
+        }
     }
 
     @Override
     public void setUpClass() throws Exception {
-        port = Network.getFreeServerPort();
-        log.info("Starting embedded Mongodb server on {} port.", port);
         doStart();
     }
 
@@ -223,12 +139,6 @@ class GoraMongodbAuthenticationTestDriver extends GoraTestDriver {
     @Override
     public void tearDownClass() {
         log.info("Shutting down mongodb server...");
-        if(started.get()) {
-            _mongod.stop();
-            _mongodExe.stop();
-            started.set(false);
-        }
+        _container.stop();
     }
-
-
 }
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java
index efd14fc..a44f70d 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/PLAIN_AuthenticationTest.java
@@ -17,21 +17,19 @@
  */
 package org.apache.gora.mongodb.authentications;
 
-import de.flapdoodle.embed.mongo.distribution.Version;
-import org.apache.gora.mongodb.GoraMongodbTestDriver;
 import org.apache.gora.mongodb.store.TestMongoStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 /**
- * Perform {@link TestMongoStore} tests on MongoDB 3.2.x server with Plain Authentication mechanism.
+ * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with Plain Authentication mechanism.
  */
 public class PLAIN_AuthenticationTest extends TestMongoStore {
   private static Logger log = LoggerFactory
           .getLogger(PLAIN_AuthenticationTest.class);
   static {
     try {
-      setTestDriver(new GoraMongodbAuthenticationTestDriver("PLAIN", Version.Main.V3_4));
+      setTestDriver(new GoraMongodbAuthenticationTestDriver("PLAIN", "3.6"));
     } catch (Exception e) {
       log.error("MongoDb Test Driver initialization failed. "+ e.getMessage());
     }
diff --git a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java
index c7fd045..c8e95fb 100644
--- a/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java
+++ b/gora-mongodb/src/test/java/org/apache/gora/mongodb/authentications/SCRAM_SHA_1_AuthenticationTest.java
@@ -17,20 +17,15 @@
  */
 package org.apache.gora.mongodb.authentications;
 
-import de.flapdoodle.embed.mongo.distribution.Version;
 import org.apache.gora.mongodb.store.TestMongoStore;
 
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-
 /**
- * Perform {@link TestMongoStore} tests on MongoDB 3.2.x server with SCRAM-SHA-1 Authentication mechanism
+ * Perform {@link TestMongoStore} tests on MongoDB 3.6.x server with SCRAM-SHA-1 Authentication mechanism
  */
 public class SCRAM_SHA_1_AuthenticationTest extends TestMongoStore {
   static {
     try {
-      setTestDriver(new GoraMongodbAuthenticationTestDriver("SCRAM-SHA-1", Version.Main.V3_4));
+      setTestDriver(new GoraMongodbAuthenticationTestDriver("SCRAM-SHA-1", "3.6"));
     } catch (Exception e) {
       log.error("MongoDb Test Driver initialization failed. "+ e.getMessage());
     }