[GitHub] rabbah closed pull request #1928: Couchdb Docker Container for OpenWhisk

[GitBox <gi...@apache.org>]

rabbah closed pull request #1928: Couchdb Docker Container for OpenWhisk
URL: https://github.com/apache/incubator-openwhisk/pull/1928
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/.gitignore b/.gitignore
index 4fb8a00bb3..bf813fb5b3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,6 +27,7 @@ bin/
 .gradle
 build/
 !/tools/build/
+tools/db/docker/import
 
 # Python
 .ipynb_checkpoints/
diff --git a/settings.gradle b/settings.gradle
index 3d7fb394db..9ee6ec5881 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -12,6 +12,7 @@ include 'core:swift3Action'
 include 'core:javaAction'
 
 include 'tools:cli'
+include 'tools:db:docker'
 
 include 'sdk:docker'
 
diff --git a/tools/db/docker/Dockerfile b/tools/db/docker/Dockerfile
new file mode 100644
index 0000000000..c826fa9c85
--- /dev/null
+++ b/tools/db/docker/Dockerfile
@@ -0,0 +1,10 @@
+FROM couchdb:1.6
+
+RUN apt-get update && apt-get install jq
+
+COPY import /import
+COPY import.sh /import.sh
+COPY init.sh /init.sh
+
+ENTRYPOINT ["tini", "--", "/init.sh"]
+CMD ["couchdb"]
\ No newline at end of file
diff --git a/tools/db/docker/build.gradle b/tools/db/docker/build.gradle
new file mode 100644
index 0000000000..ffee2efe32
--- /dev/null
+++ b/tools/db/docker/build.gradle
@@ -0,0 +1,114 @@
+ext {
+    dockerImageName = 'couchdb'
+    couchdbImage = 'couchdb:1.6'
+    couchdbUser = "gradle_admin"
+    couchdbPass = "gradle_password"
+    couchdbPrefix = "local_"
+    ansibleImage = 'williamyeh/ansible:debian8'
+    tmp_src = file(gradle.gradleUserHomeDir.getPath() + '/openwhisk').getPath()
+    import_src = file(project.rootDir.getPath() + '/tools/db/docker/import').getPath()
+}
+
+apply from: '../../../gradle/docker.gradle'
+
+/**
+ * Creates a temporary folder used by this build with the sources.
+ * This is needed to make sure the project is in the user's home folder
+ * which is mounted by docker by default
+ */
+task setupTmpFolder(type: Copy) {
+    new File(tmp_src.toString()).mkdirs()
+    new File(import_src.toString()).mkdirs()
+    println 'Copying from:' + project.rootDir.getPath() + ' into the temporary folder:' + tmp_src
+    from project.rootDir
+    include 'tools/db/**/*', 'ansible/**/*'
+    into tmp_src
+}
+
+/**
+ * Creates one container with CouchDB and executes Ansible playbooks to init DB.
+ * Ansible is executed using another container so that the build is not dependent on Ansible.
+ */
+task exportCleanDB() {
+    doFirst {
+        // start a docker container with CouchDB
+        def db_cmd = dockerBinary + ['run', '-d', '--name=gradle_couchdb', '--expose', '5984'] +
+                ['-e', 'COUCHDB_USER=' + couchdbUser, '-e', 'COUCHDB_PASSWORD=' + couchdbPass] +
+                ['-v', tmp_src + ':/openwhisk'] +
+                [couchdbImage]
+
+        // start a docker container to run Ansible playbook that configures CouchDB
+        def ansible_cmd = dockerBinary + ['run', '--name=gradle_ansible', '--rm'] +
+                ['-v', tmp_src + ':/openwhisk', '-w', '/openwhisk/ansible'] +
+                ['--link=gradle_couchdb', '-t', ansibleImage] +
+                ['ansible-playbook', 'setup.yml', 'initdb.yml', 'wipe.yml'] +
+                ['-e', 'db_username=' + couchdbUser, '-e', 'db_password=' + couchdbPass] +
+                ['-e', 'db_host=gradle_couchdb', '-e', 'openwhisk_home=/openwhisk', '-e', 'db_prefix=' + couchdbPrefix]
+
+        println("${new Date()}: Executing '${db_cmd.join(" ")}'")
+        println("${new Date()}: Executing '${ansible_cmd.join(" ")}'")
+        def timeout = 60
+        def db_proc = db_cmd.execute()
+        db_proc.consumeProcessOutput(System.out, System.err)
+        db_proc.waitForOrKill(timeout * 1000)
+        def ansible_proc = ansible_cmd.execute()
+        ansible_proc.consumeProcessOutput(System.out, System.err)
+        ansible_proc.waitForOrKill(timeout * 1000)
+
+        // dump DBs: local_whisks and subjects
+        // execute a curl command inside the CouchDB container and save the output on a shared volume
+        def dump_dbs = dockerBinary + ['exec', 'gradle_couchdb'] +
+                ['curl', 'http://127.0.0.1:5984/{' + couchdbPrefix + 'whisks,subjects}/_all_docs?include_docs=true&attachments=true', '-o', '/openwhisk/couchdb_#1.json']
+
+        println("${new Date()}: Executing '${dump_dbs.join(" ")}'")
+        def dump_dbs_proc = dump_dbs.execute()
+        dump_dbs_proc.consumeProcessOutput(System.out, System.err)
+        dump_dbs_proc.waitForOrKill(timeout * 1000)
+
+        // stop the containers
+        def docker_stop_proc = (dockerBinary + ['stop', 'gradle_couchdb']).execute()
+        docker_stop_proc.consumeProcessOutput(System.out, System.err)
+        docker_stop_proc.waitForOrKill(timeout * 1000)
+        docker_stop_proc = (dockerBinary + ['rm', 'gradle_couchdb']).execute()
+        docker_stop_proc.consumeProcessOutput(System.out, System.err)
+        docker_stop_proc.waitForOrKill(timeout * 1000)
+        docker_stop_proc = (dockerBinary + ['stop', 'gradle_ansible']).execute()
+        docker_stop_proc.consumeProcessOutput(System.out, System.err)
+        docker_stop_proc.waitForOrKill(timeout * 1000)
+    }
+
+}
+
+task avoidRebuild(type: Copy) {
+    from project.rootDir
+    include 'tools/db/**/*', 'ansible/**/*'
+    into tmp_src
+}
+
+task backupDB(type: Copy) {
+    doLast {
+        println 'Getting db dump from ' + tmp_src
+        from tmp_src
+        include 'couchdb_*.json'
+        into 'import'
+    }
+}
+
+task deleteTmpSrc(type: Delete) {
+    delete tmp_src
+    followSymlinks = true
+}
+
+task deleteDBBackup(type: Delete) {
+    delete import_src
+}
+
+exportCleanDB.dependsOn setupTmpFolder
+exportCleanDB.onlyIf { setupTmpFolder.didWork }
+exportCleanDB.finalizedBy deleteTmpSrc
+
+backupDB.dependsOn exportCleanDB
+backupDB.finalizedBy avoidRebuild
+
+distDocker.dependsOn backupDB
+distDocker.finalizedBy deleteDBBackup
diff --git a/tools/db/docker/import.sh b/tools/db/docker/import.sh
new file mode 100755
index 0000000000..c48c241ef7
--- /dev/null
+++ b/tools/db/docker/import.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+#
+# This script looks inside the /import folder to see if there are exported JSON documents to be imported.
+# NOTE: it only creates the Database if it's not created already.
+#
+
+#2. wait for the DB to start
+echo "waiting for the database to come up ... "
+until $(curl --output /dev/null --silent --head --fail http://127.0.0.1:5984/_all_dbs); do printf '.'; sleep 5; done
+
+#3. import from /import if DB is empty
+echo "import databases from the /import folder"
+
+cd /import
+# rename couchdb_{DB_NAME}.json in {DB_NAME}
+for f in couchdb*.json; do mv "$f" "`echo $f | cut -c 9- | cut -d'.' -f 1`"; done
+
+# check if DB exists by marking all the DBs that are found with _exists suffix
+for f in *; do mv "$f" "`if [ $(curl -s -o /dev/null -i -w "%{http_code}" http://localhost:5984/$f) -eq 200 ]; then echo ${f}_exists; else echo $f; fi`"; done
+echo "DB status ..."
+ls -la
+echo "Removing DBs that exist ... "
+rm -rf *_exists
+
+# at this point we only have the DBs that don't exist and we can create them
+echo "Fixing imports as per https://wiki.apache.org/couchdb/HTTP_Bulk_Document_API ..."
+for f in *; do cat ${f} | jq '{"docs": [.rows[].doc]}' | jq 'del(.docs[]._rev)' > ${f}.tmp; mv ${f}.tmp ${f}; done
+ls -la
+echo "Creating DBs ..."
+find ./ -type f -exec curl -X PUT -u ${COUCHDB_USER}:${COUCHDB_PASSWORD}  http://127.0.0.1:5984/{} \;
+echo "Importing DBs ..."
+find ./ -type f -exec curl -d @{} -u ${COUCHDB_USER}:${COUCHDB_PASSWORD} -H "Content-Type:application/json" http://127.0.0.1:5984/{}/_bulk_docs \;
\ No newline at end of file
diff --git a/tools/db/docker/init.sh b/tools/db/docker/init.sh
new file mode 100755
index 0000000000..afe464a9d6
--- /dev/null
+++ b/tools/db/docker/init.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+#
+# This script wraps the default one that comes with couchdb (docker-entrypoint.sh)
+# It starts an import process in the background executing /import.sh which waits for the DB to become ready
+# and then it imports data from the /import folder. For more details see the file import.sh
+#
+
+if [ "$1" = 'couchdb' ]; then
+    #1. prepare import
+    sh +x /import.sh 2>&1 /dev/stderr &
+    #2. start database
+    /docker-entrypoint.sh "$@"
+else
+    exec "$@"
+fi
+
+
+
+


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services