[GitHub] rabbah closed pull request #140: use runtimesManifest to drive invoker 'docker pull'

[GitBox <gi...@apache.org>]

rabbah closed pull request #140: use runtimesManifest to drive invoker 'docker pull'
URL: https://github.com/apache/incubator-openwhisk-deploy-kube/pull/140
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/docker/README.md b/docker/README.md
index 2356a94..7be3b0a 100644
--- a/docker/README.md
+++ b/docker/README.md
@@ -8,8 +8,11 @@ The built images are:
   * couchdb - creates and initializes a CouchDB instance for
     dev/testing of OpenWhisk.  This image is not intended for
     production usage.
-  * routemgmt - installs OpenWhisk's route management package
-    in the system namespace of the OpenWhisk deployment.
+  * docker-pull - performs a 'docker pull' for action runtimes
+    specified in runtimesManifest format -- used to prefetch
+    action runtime images for invoker nodes
   * openwhisk-catalog - installs the catalog from the project
     incubator-openwhisk-calalog to the system namespace of the
     OpenWhisk deployment.
+  * routemgmt - installs OpenWhisk's route management package
+    in the system namespace of the OpenWhisk deployment.
diff --git a/docker/docker-pull/Dockerfile b/docker/docker-pull/Dockerfile
new file mode 100644
index 0000000..90925d3
--- /dev/null
+++ b/docker/docker-pull/Dockerfile
@@ -0,0 +1,21 @@
+from ubuntu:latest
+
+ENV DOCKER_VERSION 1.12.0
+
+RUN apt-get -y update && apt-get -y install \
+  wget \
+  python-pip
+
+# Install docker client
+RUN wget --no-verbose https://get.docker.com/builds/Linux/x86_64/docker-${DOCKER_VERSION}.tgz && \
+tar --strip-components 1 -xvzf docker-${DOCKER_VERSION}.tgz -C /usr/bin docker/docker && \
+rm -f docker-${DOCKER_VERSION}.tgz && \
+chmod +x /usr/bin/docker
+
+RUN pip install --upgrade setuptools
+RUN pip install ansible==2.3.0.0
+RUN pip install jinja2==2.9.6
+
+COPY pull_images.yml /pull_images.yml
+
+CMD ["/usr/local/bin/ansible-playbook", "/pull_images.yml"]
diff --git a/docker/docker-pull/pull_images.yml b/docker/docker-pull/pull_images.yml
new file mode 100644
index 0000000..2e1a3f8
--- /dev/null
+++ b/docker/docker-pull/pull_images.yml
@@ -0,0 +1,33 @@
+---
+
+# Playbook to prefetch runtime action images from a docker registry for an invoker node.
+
+- hosts: localhost
+  vars:
+    docker_pull_retries: 10
+    docker_pull_delay: 10
+    docker_registry:
+    runtimesManifest: "{{ lookup('env', 'RUNTIMES_MANIFEST') | from_json }}"
+
+  tasks:
+    - name: docker login
+      docker_login:
+        registry: "{{ docker_registry }}"
+        username: "{{ docker_registry_username }}"
+        password: "{{ docker_registry_password }}"
+      when: docker_registry != "" and docker_registry_password is defined
+
+    - name: "pull runtime action images with tag {{runtimesManifest.defaultImageTag}}"
+      shell: "docker pull {{docker_registry}}{{runtimesManifest.defaultImagePrefix}}/{{item}}:{{runtimesManifest.defaultImageTag}}"
+      with_items: "{{ runtimesManifest.runtimes.values() | sum(start=[]) | selectattr('deprecated', 'equalto',false)  | map(attribute='image.name') | list | unique }}"
+      when: docker_registry != ""
+      retries: "{{ docker_pull_retries }}"
+      delay: "{{ docker_pull_delay }}"
+
+    - name: "pull blackboxes action images with tag {{runtimesManifest.defaultImageTag}}"
+      shell: "docker pull {{docker_registry}}{{runtimesManifest.defaultImagePrefix}}/{{item.name}}:{{runtimesManifest.defaultImageTag}}"
+      with_items:
+        - "{{ runtimesManifest.blackboxes }}"
+      when: docker_registry != ""
+      retries: "{{ docker_pull_retries }}"
+      delay: "{{ docker_pull_delay }}"
diff --git a/kubernetes/invoker/invoker.yml b/kubernetes/invoker/invoker.yml
index 7241eb5..0672a19 100644
--- a/kubernetes/invoker/invoker.yml
+++ b/kubernetes/invoker/invoker.yml
@@ -33,6 +33,21 @@ spec:
         hostPath:
           path: "/usr/lib/x86_64-linux-gnu/libapparmor.so.1"
 
+      initContainers:
+      - name: docker-pull-runtimes
+        imagePullPolicy: Always
+        image: openwhisk/kube-docker-pull
+        volumeMounts:
+        - name: dockersock
+          mountPath: "/var/run/docker.sock"
+        env:
+          # action runtimes
+          - name: "RUNTIMES_MANIFEST"
+            valueFrom:
+              configMapKeyRef:
+                name: whisk.runtimes
+                key: runtimes
+
       containers:
       - name: invoker
         imagePullPolicy: Always
@@ -52,13 +67,6 @@ spec:
           mountPath: "/containers"
         - name: apparmor
           mountPath: "/usr/lib/x86_64-linux-gnu/libapparmor.so.1"
-        lifecycle:
-          postStart:
-            exec:
-              command:
-              - "/bin/bash"
-              - "-c"
-              - "docker pull openwhisk/nodejsactionbase && docker pull openwhisk/nodejs6action && docker pull openwhisk/dockerskeleton && docker pull openwhisk/python2action && docker pull openwhisk/python3action && docker pull openwhisk/action-swift-v3.1.1 && docker pull openwhisk/swift3action && docker pull openwhisk/java8action"
         env:
           - name: "PORT"
             value: "8080"
diff --git a/tools/travis/collect-logs.sh b/tools/travis/collect-logs.sh
index 56a4fe3..0798c5f 100755
--- a/tools/travis/collect-logs.sh
+++ b/tools/travis/collect-logs.sh
@@ -17,7 +17,8 @@ kubectl -n openwhisk logs -lname=zookeeper >& logs/zookeeper.log
 kubectl -n openwhisk logs -lname=kafka >& logs/kafka.log
 kubectl -n openwhisk logs controller-0 >& logs/controller-0.log
 kubectl -n openwhisk logs controller-1 >& logs/controller-1.log
-kubectl -n openwhisk logs -lname=invoker >& logs/invoker.log
+kubectl -n openwhisk logs -lname=invoker -c docker-pull-runtimes >& logs/invoker-docker-pull.log
+kubectl -n openwhisk logs -lname=invoker -c invoker >& logs/invoker-invoker.log
 kubectl -n openwhisk logs -lname=nginx >& logs/nginx.log
 kubectl -n openwhisk logs jobs/install-routemgmt >& logs/routemgmt.log
 kubectl -n openwhisk logs jobs/install-catalog >& logs/catalog.log
diff --git a/tools/travis/deploy.sh b/tools/travis/deploy.sh
index 3a10cd7..11d75a0 100755
--- a/tools/travis/deploy.sh
+++ b/tools/travis/deploy.sh
@@ -10,8 +10,11 @@ cd $ROOTDIR
 echo "Publishing kube-couchdb image"
 ./tools/travis/publish.sh openwhisk kube-couchdb latest docker/couchdb
 
-echo "Publishing kube-routemgmt image"
-./tools/travis/publish.sh openwhisk kube-routemgmt latest docker/routemgmt
+echo "Publishing kube-docker-pull image"
+./tools/travis/publish.sh openwhisk kube-docker-pull latest docker/docker-pull
 
 echo "Publishing kube-openwhisk-catalog image"
 ./tools/travis/publish.sh openwhisk kube-openwhisk-catalog latest docker/openwhisk-catalog
+
+echo "Publishing kube-routemgmt image"
+./tools/travis/publish.sh openwhisk kube-routemgmt latest docker/routemgmt


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services