You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ra...@apache.org on 2017/10/17 21:56:39 UTC
[incubator-openwhisk] branch master updated: Move docker runtime
into its own repo (#2850)
This is an automated email from the ASF dual-hosted git repository.
rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk.git
The following commit(s) were added to refs/heads/master by this push:
new faeee10 Move docker runtime into its own repo (#2850)
faeee10 is described below
commit faeee1009cdb317fe0e6fcaf5100f1c940ff360d
Author: Carlos Santana <cs...@gmail.com>
AuthorDate: Tue Oct 17 17:56:37 2017 -0400
Move docker runtime into its own repo (#2850)
---
ansible/edge.yml | 1 -
ansible/roles/nginx/templates/nginx.conf.j2 | 6 +-
ansible/roles/sdk/tasks/clean.yml | 8 -
ansible/roles/sdk/tasks/deploy.yml | 39 ---
ansible/roles/sdk/tasks/main.yml | 10 -
core/actionProxy/Dockerfile | 23 +-
core/actionProxy/actionproxy.py | 271 ---------------------
core/actionProxy/delete-build-run.sh | 23 --
core/actionProxy/stub.sh | 9 -
sdk/docker/Dockerfile | 16 +-
sdk/docker/README.md | 34 ---
sdk/docker/build.gradle | 2 -
sdk/docker/buildAndPush.sh | 24 --
sdk/docker/example.c | 17 --
.../src/test/scala/system/basic/WskSdkTests.scala | 9 -
tools/cli/go-whisk-cli/commands/sdk.go | 2 +-
16 files changed, 8 insertions(+), 486 deletions(-)
diff --git a/ansible/edge.yml b/ansible/edge.yml
index cf80ebe..6566d57 100644
--- a/ansible/edge.yml
+++ b/ansible/edge.yml
@@ -8,4 +8,3 @@
roles:
- nginx
- cli
- - sdk
diff --git a/ansible/roles/nginx/templates/nginx.conf.j2 b/ansible/roles/nginx/templates/nginx.conf.j2
index 31c1ccf..6a0b4dc 100644
--- a/ansible/roles/nginx/templates/nginx.conf.j2
+++ b/ansible/roles/nginx/templates/nginx.conf.j2
@@ -76,8 +76,12 @@ http {
proxy_read_timeout 70s; # 60+10 additional seconds to allow controller to terminate request
}
+ location /blackbox.tar.gz {
+ return 301 https://github.com/apache/incubator-openwhisk-runtime-docker/releases/download/sdk%400.1.0/blackbox-0.1.0.tar.gz;
+ }
+ # leaving this for a while for clients out there to update to the new endpoint
location /blackbox-0.1.0.tar.gz {
- root /etc/nginx;
+ return 301 /blackbox.tar.gz;
}
location /OpenWhiskIOSStarterApp.zip {
diff --git a/ansible/roles/sdk/tasks/clean.yml b/ansible/roles/sdk/tasks/clean.yml
deleted file mode 100644
index c9e83ed..0000000
--- a/ansible/roles/sdk/tasks/clean.yml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-# Remove SDK artifacts from nginx.
-
-- name: remove blackbox sdk
- file:
- path: "{{ nginx.confdir }}/blackbox-0.1.0.tar.gz"
- state: absent
- become: "{{ sdk.dir.become }}"
diff --git a/ansible/roles/sdk/tasks/deploy.yml b/ansible/roles/sdk/tasks/deploy.yml
deleted file mode 100644
index 4b6b2de..0000000
--- a/ansible/roles/sdk/tasks/deploy.yml
+++ /dev/null
@@ -1,39 +0,0 @@
----
-# Tasks for handling SDK generation and publishing
-
-- name: ensure nginx config directory exists
- file:
- path: "{{ nginx.confdir }}"
- state: directory
- become: "{{ sdk.dir.become }}"
-
-# Blackbox
-
-- name: make temp dir
- local_action: shell "mktemp" "-d" "{{ lookup('env', 'TMPDIR') | default('/tmp', true) }}/whisk.XXXXXXXX"
- register: tmpdir
-
-- name: copy docker sdk to dockerSkeleton in scratch space
- local_action: copy src="{{ openwhisk_home }}/sdk/docker/{{ item }}" dest="{{ tmpdir.stdout }}/dockerSkeleton/"
- with_items:
- - buildAndPush.sh
- - Dockerfile
- - example.c
- - README.md
-
-- name: rename base image in Dockerfile
- local_action: replace dest="{{ tmpdir.stdout }}/dockerSkeleton/Dockerfile" regexp='^FROM dockerskeleton*.*$' replace='FROM openwhisk/dockerskeleton'
-
-- name: fix file permissions
- local_action: file path="{{ tmpdir.stdout }}/dockerSkeleton/buildAndPush.sh" mode=0755
-
-- name: build blackbox container artifact
- local_action: shell "tar" "--exclude=build.gradle" "-czf" "{{ tmpdir.stdout }}/blackbox-0.1.0.tar.gz" "dockerSkeleton" chdir="{{ tmpdir.stdout }}"
-
-- name: copy blackbox container artifact to nginx
- copy:
- src: "{{ tmpdir.stdout }}/blackbox-0.1.0.tar.gz"
- dest: "{{ nginx.confdir }}"
-
-- name: remove temp dir
- local_action: file path="{{ tmpdir.stdout }}" state=absent
diff --git a/ansible/roles/sdk/tasks/main.yml b/ansible/roles/sdk/tasks/main.yml
deleted file mode 100644
index 32f892d..0000000
--- a/ansible/roles/sdk/tasks/main.yml
+++ /dev/null
@@ -1,10 +0,0 @@
----
-# This role will build and publish Openwhisk SDKs. Currently covers blackbox starter kit.
-# In deploy mode it will generate downloadable artifacts and copy them to nginx.
-# In clean mode it will remove the artifacts from nginx.
-
-- include: deploy.yml
- when: mode == "deploy"
-
-- include: clean.yml
- when: mode == "clean"
diff --git a/core/actionProxy/Dockerfile b/core/actionProxy/Dockerfile
index 6ec4ff4..16094a7 100644
--- a/core/actionProxy/Dockerfile
+++ b/core/actionProxy/Dockerfile
@@ -1,23 +1,2 @@
# Dockerfile for docker skeleton (useful for running blackbox binaries, scripts, or Python 3 actions) .
-FROM python:3.6.1-alpine
-
-# Upgrade and install basic Python dependencies.
-RUN apk add --no-cache bash \
- && apk add --no-cache --virtual .build-deps \
- bzip2-dev \
- gcc \
- libc-dev \
- && pip install --upgrade pip setuptools six \
- && pip install --no-cache-dir gevent==1.2.1 flask==0.12 \
- && apk del .build-deps
-
-ENV FLASK_PROXY_PORT 8080
-
-RUN mkdir -p /actionProxy
-ADD actionproxy.py /actionProxy/
-
-RUN mkdir -p /action
-ADD stub.sh /action/exec
-RUN chmod +x /action/exec
-
-CMD ["/bin/bash", "-c", "cd actionProxy && python -u actionproxy.py"]
+FROM openwhisk/dockerskeleton:1.0.0
diff --git a/core/actionProxy/actionproxy.py b/core/actionProxy/actionproxy.py
deleted file mode 100644
index c68231f..0000000
--- a/core/actionProxy/actionproxy.py
+++ /dev/null
@@ -1,271 +0,0 @@
-"""Executable Python script for a proxy service to dockerSkeleton.
-
-Provides a proxy service (using Flask, a Python web microframework)
-that implements the required /init and /run routes to interact with
-the OpenWhisk invoker service.
-
-The implementation of these routes is encapsulated in a class named
-ActionRunner which provides a basic framework for receiving code
-from an invoker, preparing it for execution, and then running the
-code when required.
-
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-"""
-
-import sys
-import os
-import json
-import subprocess
-import codecs
-import flask
-from gevent.wsgi import WSGIServer
-import zipfile
-import io
-import base64
-
-
-class ActionRunner:
- """ActionRunner."""
- LOG_SENTINEL = 'XXX_THE_END_OF_A_WHISK_ACTIVATION_XXX'
-
- # initializes the runner
- # @param source the path where the source code will be located (if any)
- # @param binary the path where the binary will be located (may be the
- # same as source code path)
- def __init__(self, source=None, binary=None):
- defaultBinary = '/action/exec'
- self.source = source if source else defaultBinary
- self.binary = binary if binary else defaultBinary
-
- def preinit(self):
- return
-
- # extracts from the JSON object message a 'code' property and
- # writes it to the <source> path. The source code may have an
- # an optional <epilogue>. The source code is subsequently built
- # to produce the <binary> that is executed during <run>.
- # @param message is a JSON object, should contain 'code'
- # @return True iff binary exists and is executable
- def init(self, message):
- def prep():
- self.preinit()
- if 'code' in message and message['code'] is not None:
- binary = message['binary'] if 'binary' in message else False
- if not binary:
- return self.initCodeFromString(message)
- else:
- return self.initCodeFromZip(message)
- else:
- return False
-
- if prep():
- try:
- # write source epilogue if any
- # the message is passed along as it may contain other
- # fields relevant to a specific container.
- if self.epilogue(message) is False:
- return False
- # build the source
- if self.build(message) is False:
- return False
- except Exception:
- return False
- # verify the binary exists and is executable
- return self.verify()
-
- # optionally appends source to the loaded code during <init>
- def epilogue(self, init_arguments):
- return
-
- # optionally builds the source code loaded during <init> into an executable
- def build(self, init_arguments):
- return
-
- # @return True iff binary exists and is executable, False otherwise
- def verify(self):
- return (os.path.isfile(self.binary) and
- os.access(self.binary, os.X_OK))
-
- # constructs an environment for the action to run in
- # @param message is a JSON object received from invoker (should
- # contain 'value' and 'api_key' and other metadata)
- # @return an environment dictionary for the action process
- def env(self, message):
- # make sure to include all the env vars passed in by the invoker
- env = os.environ
- for p in ['api_key', 'namespace', 'action_name', 'activation_id', 'deadline']:
- if p in message:
- env['__OW_%s' % p.upper()] = message[p]
- return env
-
- # runs the action, called iff self.verify() is True.
- # @param args is a JSON object representing the input to the action
- # @param env is the environment for the action to run in (defined edge
- # host, auth key)
- # return JSON object result of running the action or an error dictionary
- # if action failed
- def run(self, args, env):
- def error(msg):
- # fall through (exception and else case are handled the same way)
- sys.stdout.write('%s\n' % msg)
- return (502, {'error': 'The action did not return a dictionary.'})
-
- try:
- input = json.dumps(args)
- p = subprocess.Popen(
- [self.binary, input],
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- env=env)
- except Exception as e:
- return error(e)
-
- # run the process and wait until it completes.
- # stdout/stderr will always be set because we passed PIPEs to Popen
- (o, e) = p.communicate()
-
- # stdout/stderr may be either text or bytes, depending on Python
- # version, so if bytes, decode to text. Note that in Python 2
- # a string will match both types; so also skip decoding in that case
- if isinstance(o, bytes) and not isinstance(o, str):
- o = o.decode('utf-8')
- if isinstance(e, bytes) and not isinstance(e, str):
- e = e.decode('utf-8')
-
- # get the last line of stdout, even if empty
- lastNewLine = o.rfind('\n', 0, len(o)-1)
- if lastNewLine != -1:
- # this is the result string to JSON parse
- lastLine = o[lastNewLine+1:].strip()
- # emit the rest as logs to stdout (including last new line)
- sys.stdout.write(o[:lastNewLine+1])
- else:
- # either o is empty or it is the result string
- lastLine = o.strip()
-
- if e:
- sys.stderr.write(e)
-
- try:
- json_output = json.loads(lastLine)
- if isinstance(json_output, dict):
- return (200, json_output)
- else:
- return error(lastLine)
- except Exception:
- return error(lastLine)
-
- # initialize code from inlined string
- def initCodeFromString(self, message):
- with codecs.open(self.source, 'w', 'utf-8') as fp:
- fp.write(message['code'])
- return True
-
- # initialize code from base64 encoded archive
- def initCodeFromZip(self, message):
- try:
- bytes = base64.b64decode(message['code'])
- bytes = io.BytesIO(bytes)
- archive = zipfile.ZipFile(bytes)
- archive.extractall(os.path.dirname(self.source))
- archive.close()
- return True
- except Exception as e:
- print('err', str(e))
- return False
-
-proxy = flask.Flask(__name__)
-proxy.debug = False
-runner = None
-
-
-def setRunner(r):
- global runner
- runner = r
-
-
-@proxy.route('/init', methods=['POST'])
-def init():
- message = flask.request.get_json(force=True, silent=True)
- if message and not isinstance(message, dict):
- flask.abort(404)
- else:
- value = message.get('value', {}) if message else {}
-
- if not isinstance(value, dict):
- flask.abort(404)
-
- try:
- status = runner.init(value)
- except Exception as e:
- status = False
-
- if status is True:
- return ('OK', 200)
- else:
- response = flask.jsonify({'error': 'The action failed to generate or locate a binary. See logs for details.'})
- response.status_code = 502
- return complete(response)
-
-
-@proxy.route('/run', methods=['POST'])
-def run():
- def error():
- response = flask.jsonify({'error': 'The action did not receive a dictionary as an argument.'})
- response.status_code = 404
- return complete(response)
-
- message = flask.request.get_json(force=True, silent=True)
- if message and not isinstance(message, dict):
- return error()
- else:
- args = message.get('value', {}) if message else {}
- if not isinstance(args, dict):
- return error()
-
- if runner.verify():
- try:
- code, result = runner.run(args, runner.env(message or {}))
- response = flask.jsonify(result)
- response.status_code = code
- except Exception as e:
- response = flask.jsonify({'error': 'Internal error. {}'.format(e)})
- response.status_code = 500
- else:
- response = flask.jsonify({'error': 'The action failed to locate a binary. See logs for details.'})
- response.status_code = 502
- return complete(response)
-
-
-def complete(response):
- # Add sentinel to stdout/stderr
- sys.stdout.write('%s\n' % ActionRunner.LOG_SENTINEL)
- sys.stdout.flush()
- sys.stderr.write('%s\n' % ActionRunner.LOG_SENTINEL)
- sys.stderr.flush()
- return response
-
-
-def main():
- port = int(os.getenv('FLASK_PROXY_PORT', 8080))
- server = WSGIServer(('', port), proxy, log=None)
- server.serve_forever()
-
-if __name__ == '__main__':
- setRunner(ActionRunner())
- main()
diff --git a/core/actionProxy/delete-build-run.sh b/core/actionProxy/delete-build-run.sh
deleted file mode 100755
index 430c02d..0000000
--- a/core/actionProxy/delete-build-run.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/usr/bin/env bash
-
-# Useful for local testing.
-# USE WITH CAUTION !!
-
-# This script is useful for testing the action proxy (or its derivatives)
-# in combination with [init,run].py. Use it to rebuild the container image
-# and start the proxy: delete-build-run.sh whisk/dockerskeleton.
-
-# Removes all previously built instances.
-remove=$(docker ps -a -q)
-if [[ ! -z $remove ]]; then
- docker rm $remove
-fi
-
-image=${1:-openwhisk/dockerskeleton}
-docker build -t $image .
-
-echo ""
-echo " ---- RUNNING ---- "
-echo ""
-
-docker run -i -t -p 8080:8080 $image
diff --git a/core/actionProxy/stub.sh b/core/actionProxy/stub.sh
deleted file mode 100644
index 842d00a..0000000
--- a/core/actionProxy/stub.sh
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/bin/bash
-
-echo \
-'This is a stub action that should be replaced with user code (e.g., script or compatible binary).
-The input to the action is received as an argument from the command line.
-Actions may log to stdout or stderr. By convention, the last line of output must
-be a stringified JSON object which represents the result of the action.'
-
-echo '{ "error": "This is a stub action. Replace it with custom logic." }'
\ No newline at end of file
diff --git a/sdk/docker/Dockerfile b/sdk/docker/Dockerfile
index 3fb6146..5a448e6 100644
--- a/sdk/docker/Dockerfile
+++ b/sdk/docker/Dockerfile
@@ -1,17 +1,3 @@
# Dockerfile for example whisk docker action
-FROM dockerskeleton
+FROM openwhisk/example:1.0.0
-ENV FLASK_PROXY_PORT 8080
-
-### Add source file(s)
-ADD example.c /action/example.c
-
-RUN apk add --no-cache --virtual .build-deps \
- bzip2-dev \
- gcc \
- libc-dev \
-### Compile source file(s)
- && cd /action; gcc -o exec example.c \
- && apk del .build-deps
-
-CMD ["/bin/bash", "-c", "cd actionProxy && python -u actionproxy.py"]
diff --git a/sdk/docker/README.md b/sdk/docker/README.md
deleted file mode 100644
index 4eb6158..0000000
--- a/sdk/docker/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-Blackbox Actions
-================
-
-1. Download and install the OpenWhisk CLI
-2. Install OpenWhisk Docker action skeleton.
-3. Add user code
-4. Build image
-5. Push image
-6. Test out action with CLI
-
-The script `buildAndPush.sh` is provided for your convenience. The following command sequence
-runs the included example Docker action container using OpenWhisk.
-
-```
-# install dockerSkeleton with example
-wsk sdk install docker
-
-# change working directory
-cd dockerSkeleton
-
-# build/push, argument is your docker hub user name and a valid docker image name
-./buildAndPush <dockerhub username>/whiskexample
-
-# create docker action
-wsk action create dockerSkeletonExample --docker <dockerhub username>/whiskExample
-
-# invoke created action
-wsk action invoke dockerSkeletonExample --blocking
-```
-
-The executable file must be located in the `/action` folder.
-The name of the executable must be `/action/exec` and can be any file with executable permissions.
-The sample docker action runs `example.c` by copying and building the source inside the container
-as `/action/exec` (see `Dockerfile` lines 7 and 14).
diff --git a/sdk/docker/build.gradle b/sdk/docker/build.gradle
index a8cc3d7..be57df1 100644
--- a/sdk/docker/build.gradle
+++ b/sdk/docker/build.gradle
@@ -1,4 +1,2 @@
ext.dockerImageName = 'example'
-
apply from: '../../gradle/docker.gradle'
-distDocker.dependsOn ':core:actionProxy:distDocker'
diff --git a/sdk/docker/buildAndPush.sh b/sdk/docker/buildAndPush.sh
deleted file mode 100755
index 85c75ce..0000000
--- a/sdk/docker/buildAndPush.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-#
-# This script will build the docker image and push it to dockerhub.
-#
-# Usage: buildAndPush.sh imageName
-#
-# Dockerhub image names look like "username/appname" and must be all lower case.
-# For example, "janesmith/calculator"
-
-IMAGE_NAME=$1
-echo "Using $IMAGE_NAME as the image name"
-
-# Make the docker image
-docker build -t $IMAGE_NAME .
-if [ $? -ne 0 ]; then
- echo "Docker build failed"
- exit
-fi
-docker push $IMAGE_NAME
-if [ $? -ne 0 ]; then
- echo "Docker push failed"
- exit
-fi
-
diff --git a/sdk/docker/example.c b/sdk/docker/example.c
deleted file mode 100644
index 35f61a4..0000000
--- a/sdk/docker/example.c
+++ /dev/null
@@ -1,17 +0,0 @@
-#include <stdio.h>
-
-/**
- * This is an example C program that can run as a native openwhisk
- * action using the openwhisk/dockerskeleton.
- *
- * The input to the action is received as an argument from the command line.
- * Actions may log to stdout or stderr.
- * By convention, the last line of output must be a stringified JSON object
- * which represents the result of the action.
- */
-
-int main(int argc, char *argv[]) {
- printf("This is an example log message from an arbitrary C program!\n");
- printf("{ \"msg\": \"Hello from arbitrary C program!\", \"args\": %s }",
- (argc == 1) ? "undefined" : argv[1]);
-}
\ No newline at end of file
diff --git a/tests/src/test/scala/system/basic/WskSdkTests.scala b/tests/src/test/scala/system/basic/WskSdkTests.scala
index 3e7e3b2..021e8ca 100644
--- a/tests/src/test/scala/system/basic/WskSdkTests.scala
+++ b/tests/src/test/scala/system/basic/WskSdkTests.scala
@@ -19,15 +19,12 @@ package system.basic
import java.io.File
-import scala.collection.JavaConversions.asScalaBuffer
-
import org.apache.commons.io.FileUtils
import org.junit.runner.RunWith
import org.scalatest.junit.JUnitRunner
import common.TestHelpers
import common.TestUtils.ERROR_EXIT
import common.TestUtils.SUCCESS_EXIT
-import common.WhiskProperties
import common.Wsk
import common.WskProps
import common.WskTestHelpers
@@ -78,12 +75,6 @@ class WskSdkTests extends TestHelpers with WskTestHelpers {
val buildAndPushFile = new File(sdk, "buildAndPush.sh")
buildAndPushFile.canExecute() should be(true)
-
- // confirm there is no other divergence from the base dockerfile
- val originalDockerfile = WhiskProperties.getFileRelativeToWhiskHome("sdk/docker/Dockerfile")
- val originalLines = FileUtils.readLines(originalDockerfile)
- lines.get(0) shouldBe originalLines.get(0)
- lines.drop(2).mkString("\n") shouldBe originalLines.drop(2).mkString("\n")
} finally {
FileUtils.deleteDirectory(dir)
}
diff --git a/tools/cli/go-whisk-cli/commands/sdk.go b/tools/cli/go-whisk-cli/commands/sdk.go
index c9a9e50..d6b78e6 100644
--- a/tools/cli/go-whisk-cli/commands/sdk.go
+++ b/tools/cli/go-whisk-cli/commands/sdk.go
@@ -253,6 +253,6 @@ func init() {
sdkCmd.AddCommand(sdkInstallCmd)
sdkMap = make(map[string]*sdkInfo)
- sdkMap["docker"] = &sdkInfo{ UrlPath: "blackbox-0.1.0.tar.gz", FileName: "blackbox-0.1.0.tar.gz", isGzTar: true, Unpack: true, UnpackDir: "dockerSkeleton"}
+ sdkMap["docker"] = &sdkInfo{ UrlPath: "blackbox.tar.gz", FileName: "blackbox.tar.gz", isGzTar: true, Unpack: true, UnpackDir: "dockerSkeleton"}
sdkMap["ios"] = &sdkInfo{ UrlPath: "OpenWhiskIOSStarterApp.zip", FileName: "OpenWhiskIOSStarterApp.zip", IsZip: true, Unpack: false}
}
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].