You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/03/12 12:53:34 UTC
[GitHub] jonpspri closed pull request #13: WIP: Multi-architecture build for for the nodejs actions
jonpspri closed pull request #13: WIP: Multi-architecture build for for the nodejs actions
URL: https://github.com/apache/incubator-openwhisk-runtime-nodejs/pull/13
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index 52f108d..9e3da48 100644
--- a/.gitignore
+++ b/.gitignore
@@ -35,6 +35,7 @@ bin/
.gradle
build/
!/tools/build/
+docker-local.gradle
# Python
.ipynb_checkpoints/
@@ -71,3 +72,6 @@ ansible/roles/nginx/files/*cert.pem
!tests/dat/actions/python_virtualenv_dir.zip
!tests/dat/actions/python_virtualenv_name.zip
!tests/dat/actions/zippedaction.zip
+
+# Ignore un-encrypted TLS keys
+*.pem
diff --git a/.travis.yml b/.travis.yml
index 8460f66..f20113c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,26 +1,18 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
sudo: required
group: deprecated-2017Q3
language: scala
services:
- docker
before_install:
+ # Use GPG encryption because it reduces the number of secrets that must be
+ # managed across repositories. When forking a repository as a core
+ # contributor, contact maintainer to have a newly-encrypted GPG passphrase
+ # communicated.
+- for i in ./tools/travis/tls/*/*.pem.gpg; do
+ echo "${multiarch_encryption_pwd}" | gpg --passphrase-fd 0 "$i";
+ done
+- echo "${multiarch_encryption_pwd}"| gpg --passphrase-fd 0 ./tools/travis/docker-local.gradle.gpg
+- cp ./tools/travis/docker-local.gradle .
- "./tools/travis/setup.sh"
install: true
script:
@@ -35,3 +27,5 @@ env:
global:
- secure: Y1ldwIQ6bc3/3Pc0E+qQ6K2M830B9BObYDlsNilPwF/kak3YSfF7SuXuRbJGjTdhH2KOotZD2CwONgP2yvOSPBToC/HpnXYfAGtgblrxQORvgdik88CFWa3Lli1pwlpdzKQNWhBvglzq+IIS98wqzmwqGr8zKA+Iau2ByHdb1j3M9rrIY9V6oU9Gwim1apcRyfI/as3+QfPtt8BUAl2U7+PprxwJigyF/mcZnBJbd7IjrilE2gldZLxKlBiffoKVBinrEg3IQGJPt6k8riw264pBQEpcA0ZBsPUvMaISSxLb+d1ymp3WsiTJUjv+URR/HcdDa7P9jY+ouc8PQz4Yt+Ii38lM2tQU480APfVTyfj6drkjL/+54mYuxm8TzkBWcM2j6/FYT+8HvK/pF35wDJ3El+jGq7BARXg8HVxFsZgynJnhqhWDQb3xX9fK+N4K8+ct+HlsOSa5mP5i5Yo6WRTrWrFpyxVnv9crKePCiYqA2Y8ta8Wnh0sM06nLRtDbfbDjvXPQbaZqSnL4B2Cto08YoT/W/lu8QgJ3EEFlUdDOke4kv9yoXtuE0h7+8dwOvBNMVrBis3G2EYObgR4WmWjk4loYhqT9h3jrH0/5bGLzSKc++qYW0rOZ/RB21cRSe1ILQvSzWkImUoPI8b0i5baGSDq3EjTXYr3pIXSYpQk=
- secure: DOg+FgllLbyv7nEK3JJZfO/cvXy5K0L6QI/S9EJ/ivm4XBDCw1ayhrSQXvp1tMTPbWBEIv2gomPsHghJ+hVvX3dgwYdoNz9WZaNBB6lOO9U8OQW0LBsO5Eai6grzqOP35OuKtthuyR3dGJHAZo/XjhZM/jL0z6q1kNDzdS7ASwRwHJG0rHPGVlGeolH4nAity4KNJvyAspS1FYaIj9FEC/M7UT6nJVACbr9iMt/83teF/Oo2uoFI6Pa4K3nE2NViVFibToNOM3CV8kArDPDoNJviXxQ07ZM6fNijwehZ80waiPSaxFY8PLSntQNxGyB3DbomSTCcdVvtuHHQVmZgpVdvOJE8wk3R09+nq9U9FuUWLiRYSRbF1eF48YFnssPW1jEeVSenFRADcQ37e4B21ssLvXRHpQHpPVrYBZ8ffDamS7pKtEqocX/3Syc6irxHGCpxEdhvQ0Of5AWHhzB714VCijJJQiH9J4hEKXhMeAZ5SSt4eUavEHJKMhUcJ/aaku5w1+KtiYeOso1fTRbTYkEYb1A0bSNdXlGrYRRT+N683W7+ENiQIT+hTp617L/m5WQLIHfKe3gy5qt46kHFiUL3R3YlBI5OKjnLkDnUiA+0d4SFtJC/TBuPmH1fG4isAuvggTjzCb3kUG3ysin8AW0AKaXW0eqjKa/nsHWDKmc=
+ # The secure entry immediately below is where the GPG passphrase is encrypted by Travis
+ - secure: "1rpU5pWVHvOCYcjZZyMdtn6mlokJec6/qLjahwa0m7O9cUEe14zvdmNmql9TG/lfbEupkGBGpyXLsk4gl4CktBtj+kuP9VIT1ad0lGGfM455tqDD2fk+x54U7tk5imkR9Pn6Zmi0MCpvZSzCiOds+T2U9o9/uFB7WxXSyv+mjyosg2pHrK4hldoIfnhXIfFnNCjBXyGudnCQ4JW4Q+/9cDsc/rEPFjXo377dLBBQITvkORtCYUSI7WD7lBrB5u2GwZwmsJBwh0HojKOPe1gS325lC/7/PZAuxvMSDeiGOBp0JHEt2GdfJK2I8xrApigNfqZ9HaRDbBl+7XWqLzuQm/fAnPCuZOR3fYLpJrhzqjz3AZ8gQo+d7qqkLHfJxrL89egPQrjoPDaUdwm7Gp21ZjiqL5m9IAeJmtVSb9ZoXf8nqKyAGHn/K829EbKqore/4+x/6GbTm1tksyncSBU0p1Ulsfoo1lzUza9XFZGOiW9lHDxXMwT4XHg4y60cpkyHOuo98hMprB0M4GIPrDdn8OPvhJ4qcq4CveOAZtTAqWkhI+fQAL+h2T/7fIimczZ2na9+OEwBp7AEcHfTlIqQcf++KZPGd3WSNUoMlBVyE/VlLP5CArbVERIZVCd+/MP3XmuuD8OLq8CB2/cHK1Jz+J1ERPC4g1Qpy9T2SKxJv8I="
diff --git a/README.md b/README.md
index 54c5859..15f876a 100644
--- a/README.md
+++ b/README.md
@@ -1,16 +1,16 @@
<!--
#
-# Licensed to the Apache Software Foundation (ASF) under one or more contributor
-# license agreements. See the NOTICE file distributed with this work for additional
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements. See the NOTICE file distributed with this work for additional
# information regarding copyright ownership. The ASF licenses this file to you
-# under the Apache License, Version 2.0 (the # "License"); you may not use this
-# file except in compliance with the License. You may obtain a copy of the License
+# under the Apache License, Version 2.0 (the # "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy of the License
# at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#
@@ -44,36 +44,40 @@ wsk action update myAction myAction.js --kind nodejs:8
### Local development
For Node.js 6
```
-./gradlew core:nodejs6Action:distDocker
+./gradlew core:nodejs6Action:dockerBuildImage
```
This will produce the image `whisk/nodejs6action`
For Node.js 8
```
-./gradlew core:nodejs8Action:distDocker
+./gradlew core:nodejs8Action:dockerBuildImage
```
This will produce the image `whisk/action-nodejs-v8`
+### Pushing your own image
+
+You will need to configure registry information (at least username and password) in `./docker-local.gradle`. Refer to `./docker-local.gradle.sample` for examples.
Build and Push image for Node.js 6
```
-docker login
-./gradlew core:nodejs6Action:distDocker -PdockerImagePrefix=$prefix-user -PdockerRegistry=docker.io
+./gradlew core:nodejs6Action:dockerPushImage -PdockerImagePrefix=$user_prefix
```
Build and Push image for Node.js 8
```
-docker login
-./gradlew core:nodejs8Action:distDocker -PdockerImagePrefix=$prefix-user -PdockerRegistry=docker.io
+./gradlew core:nodejs8Action:dockerPushImage -PdockerImagePrefix=$user_prefix
```
Then create the action using your image from dockerhub
```
wsk action update myAction myAction.js --docker $user_prefix/nodejs6action
```
-The `$user_prefix` is usually your dockerhub user id.
+The `$user_prefix` is usually your dockerhub user id. (Officially, Docker
+registry specs call it the 'library' of your image.)
-Deploy OpenWhisk using ansible environment that contains the kind `nodejs:6` and `nodejs:8`
-Assuming you have OpenWhisk already deployed locally and `OPENWHISK_HOME` pointing to root directory of OpenWhisk core repository.
+Deploy OpenWhisk using ansible environment that contains the kind `nodejs:6`
+and `nodejs:8`.
+Assuming you have OpenWhisk already deployed locally and `OPENWHISK_HOME`
+pointing to root directory of OpenWhisk core repository.
Set `ROOTDIR` to the root directory of this repository.
@@ -94,37 +98,30 @@ ln -s ${ROOTDIR}/ansible/environments/local ${OPENWHISK_HOME}/ansible/environmen
wskdev fresh -t local-nodejs
```
-### Testing
-Install dependencies from the root directory on $OPENWHISK_HOME repository
-```
-./gradlew install
-```
+### Multi-architecture image builds
-Using gradle for the ActionContainer tests you need to use a proxy if running on Mac, if Linux then don't use proxy options
-You can pass the flags `-Dhttp.proxyHost=localhost -Dhttp.proxyPort=3128` directly in gradle command.
-Or save in your `$HOME/.gradle/gradle.properties`
-```
-systemProp.http.proxyHost=localhost
-systemProp.http.proxyPort=3128
-```
-Using gradle to run all tests
-```
-./gradlew :tests:test
-```
-Using gradle to run some tests
-```
-./gradlew :tests:test --tests *ActionContainerTests*
-```
-Using IntelliJ:
-- Import project as gradle project.
-- Make sure working directory is root of the project/repo
-- Add the following Java VM properties in ScalaTests Run Configuration, easiest is to change the Defaults for all ScalaTests to use this VM properties
+Docker supports multi-architecture manifests -- essentially "images" that pick
+the appropriate download binaries depending on the architecture of the target
+docker container. Building multi-architecture OpenWhisk images supports
+execution of OpenWhisk technology on multiple target platforms.
+
+The `./docker-local.gradle` file is where the target architectures are defined.
+Each architecture is linked to a docker instance where the image build can take
+place. That instance can be local or remote (non-TLS or TLS). TLS-secured
+instances should be secured as documented
+[by Docker](https://docs.docker.com/engine/security/https/).
+
+You'll also need to define registry information in `./docker-local.gradle`.
+The build process creates individual single-architecture images, tags them as
+`latest-${arch}`, pushes them to the registry, then creates and pushes the
+multi-architecture manifest with a tag of simply `latest`.
+
+To make the magic happen:
```
--Dhttp.proxyHost=localhost
--Dhttp.proxyPort=3128
+./gradlew core:nodejs8Action:putMultidocker -PdockerImagePrefix=$user_prefix
```
+(The `user_prefix` will default to `whisk` if it isn't set to something).
+
# License
[Apache 2.0](LICENSE.txt)
-
-
diff --git a/build.gradle b/build.gradle
index 300a1ce..cc7bc66 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1,9 +1,311 @@
+/*
+ NOTE TO MAINTAINERS:
+
+ You should not need to change this file to configure your specific
+ action image. Image specific configurations occur in:
+
+ ./docker-local.gradle -- for secrets/configurations that should NOT
+ be uploaded to GitHub for the world to see
+ ./settings.grade -- for listing the projects (and imageNames) to build
+ ./.../build.grade -- for subproject-specific configurations
+
+ Note: The writing is on the wall that as other project builds come into
+ play this module may need to move, perhaps to core/build.gradle or
+ to an included directory. But we shall persevere for the time
+ being in keeping it here where it's obvious that a lot is going on.
+ */
+
buildscript {
repositories {
jcenter()
+ maven {
+ url "http://dl.bintray.com/jonpspri/gradle-plugins/"
+ }
}
dependencies {
classpath "cz.alenkacz:gradle-scalafmt:${gradle.scalafmt.version}"
+ classpath 'org.ajoberstar:grgit:2.1.0'
+ classpath 'com.bmuschko:gradle-docker-plugin:3.2.2'
+ classpath 'com.xanophis:gradle-fatmanifest-plugin:0.0.5'
+
+ /*
+ Leaving this here to demonstrate how plugin development and
+ patching can be integrated.
+ */
+ //classpath files('../gradle-multidocker-plugin/build/libs/gradle-multidocker-plugin-0.0.5.jar')
+ }
+}
+
+import com.bmuschko.gradle.docker.DockerRegistryCredentials
+import com.bmuschko.gradle.docker.tasks.DockerInfo
+import com.bmuschko.gradle.docker.tasks.image.*
+
+// There are some serious naming mismatchees to be dealt with in v0.0.6 of gradle-fatmanifest-plugin :(
+import com.s390x.gradle.multidocker.tasks.*
+
+/*
+ So you called a registry method without providing any registry
+ credentials? Shame on you! This warning (which likely needs editing) will
+ put you back on the straight and narrow.
+
+ Located here in the code so it's available within all necessary blocks.
+ */
+private def warnNoRegistryCredentials() {
+ logger.warning """
+ |You are attempting a registry task without having provided registry
+ |credentials. Registry credentials should be provided in the
+ |'./docker-local.gradle' file. See './docker-local.gradle.sample' for
+ |an example. Meanwhile, this action does nothing.
+ |""".stripMargin()
+}
+
+/*
+ These two lists will hold the projects to be managed throughout the
+ remainder of the process -- the 'top-level' builds and the individual
+ architecture builds created beneath each of them.
+
+ Note: The lists could possible be created by re-constructing the project
+ paths, which in a performance-oriented system I'd consider doing. But
+ the findAll based approach works for now, and provides flexibility if we
+ add later capability to settings to gradle to select only certain builds
+ for certain projects (more of a matrix)
+ */
+Collection<Project> buildProjects = subprojects.findAll { subproject ->
+ gradle.dockerBuildProjects.containsKey(subproject.path)
+ }
+
+Collection<Project> individualArchProjects = buildProjects.collectMany { buildProject ->
+ buildProject.subprojects.findAll() {
+ gradle.architectures.containsKey(it.name)
+ }
+ }
+
+/*
+ Every project in ':core:* is some form of Docker build for the action image
+ and is therefore assumed to need to be able to push a Manifest List for
+ eventual publication. This block activates the Docker and Manifest List
+ plugins and configures them from information provided in the docker-local
+ file. Recall that the docker-local file was loaded in
+ settings.gradle and applied as extension properties to the gradle object.
+
+ The :core:* projects (parent projects) should all have been set up and
+ logged in 'grade.dockerBuildProjects' (by settings.gradle), along with the
+ image name for each. This is where we pull that image name and configure it
+ into the project.
+
+ Why put it in a gradle extension property first? Frankly, we don't want
+ to wait for it to be set in a project-specific build.gradle, and we want
+ to limit configuration to the project-specific build.gradle and to the
+ settings.gradle. This (root) build.gradle should rarely have to change.
+
+ (Side note: Maybe these need to be in a plugin or sub-file, but IMO that
+ actually makes maintenance more difficult.)
+ */
+
+configure (buildProjects) {
+ buildscript.repositories { jcenter() }
+
+ logger.info ("Configuring subproject '${path}'")
+
+ /*
+ We only configure Manifest List processing if registryCredentials exist,
+ because otherwise what's the point?
+ */
+ if (gradle?.registryCredentials) {
+ apply plugin: 'com.s390x.gradle.multidocker'
+
+ logger.debug "Applying registry credentials to ${it.path}"
+ multidocker.registry('default',gradle.registryCredentials)
+ }
+
+ /*
+ Retrieve project properties from the Map created in settings.gradle
+ and apply to the projects. Now the properties are available during
+ configuration in the remainder of this script.
+ */
+ logger.info "Setting properties for ${path}"
+ gradle.dockerBuildProjects[path].each { k,v -> ext.set(k,v) }
+ ext.dockerRegistry = gradle.dockerRegistry
+ logger.info "Project ${path} has dockerImageName of ${dockerImageName}"
+}
+
+/*
+ Each architecture gets its own project sharing a project directory with the
+ root project. It's EXPECTED (but not required) for the local build scripts
+ to override the build directory with architecture-specific build directories
+ as needed, which will also ease maintenance of the source tree for docker
+ builds.
+
+ Notice that in the block, we ensure that parent tasks are dependent on
+ the underlying architecture tasks. That way, we can still
+ './gradlew core:<image>:<task>' and all architectures will be built.
+ */
+
+configure (individualArchProjects) {
+ buildscript.repositories { jcenter() }
+
+ logger.info ("Configuring subproject '${path}'")
+
+ // Since this is an architecture-specific subproject, we can copy all the
+ // properties from the parent project to make our code more readable
+ parent.ext.properties.each { k,v -> ext.set(k,v) }
+
+ apply plugin: 'com.bmuschko.docker-remote-api'
+ docker {
+ /*
+ Since we survived the findAll, we know this architecture exists,
+ which means fewer guard clauses. Hooray!
+ */
+ if (gradle.architectures[name]?.url) {
+ // Don't overwrite a default if it was defined!
+ url = gradle.architectures[name].url
+ }
+ if (gradle.architectures[name]?.certPath) {
+ certPath = gradle.architectures[name].certPath
+ }
+
+ /*
+ Yet another place to check whether registryCredentials were
+ actually provided. It could be simplified by providing meaningless
+ defaults, but it's a bit clearer this way that there's a
+ responsiblity for the runtime developer.
+ */
+ if (gradle?.registryCredentials) registryCredentials {
+ url = gradle.registryCredentials.url
+ username = gradle.registryCredentials.username
+ password = gradle.registryCredentials.password
+ email = gradle.registryCredentials.email
+ }
+ }
+
+ task dockerInfo(type: DockerInfo) {
+ onNext {
+ logger.quiet "Docker Info retreived for project ${project.name}:"
+ logger.quiet " OSType = ${it.osType}"
+ logger.quiet " Architecture = ${it.architecture}"
+ owner.ext.dockerInfo = it
+ }
+ }
+ (parent.tasks.find() {it.name=='dockerInfo'} ?: parent.task('dockerInfo'))\
+ .dependsOn dockerInfo
+
+ task dockerBuildImage(type: DockerBuildImage) {
+ /*
+ Recommended default for inputDir
+ */
+ inputDir = file("${project.buildDir}/docker/${project.name}")
+
+ /*
+ The tags list is out of hand and is doing a bang-up job of making
+ my local docker repository messy. However, I haven't really
+ gotten a handle on what's needed and what's not. For example,
+ the dockerRegistry tag should probably be conditional. The
+ prefixed tag seems critical, however, to working with local-build
+ openwhisk deployments.
+ */
+
+ // Is this an appropriate prefix? Perhaps we should use 'whisk' as the
+ // default local-build prefix? Again -- not sure I have a sense of tags.
+ def prefix = findProperty('dockerImagePrefix') ?: 'openwhisk'
+ def tag = findProperty('dockerImageTag') ?: 'latest'
+
+ // Note that project.name will be the docker engine name from settings.gradle
+ // We could set up an option to declare suffixes in docker.gradle, but so far
+ // this works.
+ tags = [ // Return a list of tags build from the environment
+ "${parent.dockerImageName}:${tag}",
+ "${parent.dockerImageName}:${tag}-${project.name}",
+ "${prefix}/${parent.dockerImageName}:${tag}-${project.name}"
+ ] as String[]
+
+ // This tag is created to support uploads if there's a target registry.
+ // Note that the script DOES NOT support multiple target registries.
+ // As of the writing, there was no need to.
+ if (dockerRegistry) {
+ tags += new String("${dockerRegistry}/${prefix}/"+
+ "${parent.dockerImageName}:${tag}-${project.name}")
+ }
+ }
+ (parent.tasks.find() {it.name=='dockerBuildImage'} ?: parent.task('dockerBuildImage'))\
+ .dependsOn dockerBuildImage
+
+ /*
+ Pushing images and getting manifests only makes sense if registry
+ credentials were provided, we we make this block conditional. If
+ no credentials were provided, the tasks will instead display a warning
+ and point to a tutorial on docker-local.gradle.
+ */
+ if (gradle?.registryCredentials) {
+ task dockerPushImage(type: DockerPushImage, dependsOn: dockerBuildImage) {
+ imageName = "${dockerRegistry}/" +
+ "${findProperty('dockerImagePrefix') ?: 'openwhisk'}/" +
+ "${parent.dockerImageName}"
+ tag = "${findProperty('dockerImageTag') ?: 'latest'}-${project.name}"
+ }
+ (parent.tasks.find() {it.name=='dockerPushImage'} ?: parent.task('dockerPushImage'))\
+ .dependsOn dockerPushImage
+
+ task getImageManifest(type: GetImageManifest, dependsOn: [ dockerPushImage, dockerInfo ] ) {
+
+ targetRegistry = 'default'
+
+ imageName =
+ "${findProperty('dockerImagePrefix') ?: 'openwhisk'}/${parent.dockerImageName}"
+ tag = "${findProperty('dockerImageTag') ?: 'latest'}-${project.name}"
+
+ // Store the relevant details of the received manifest as an
+ // extension of the task.
+ onNext {
+ logger.quiet "Manifest retreived for ${imageName}:${tag}"
+ logger.quiet " Digest: ${it.digest}"
+ logger.quiet " Media Type: ${it.mediaType}"
+ logger.quiet " Size: ${it.size}"
+
+ owner.ext.manifestMap = [
+ mediaType: it.mediaType,
+ digest: it.digest,
+ size: it.size,
+ os: project.dockerInfo.dockerInfo.osType,
+ architecture: project.dockerInfo.dockerInfo.architecture
+ ]
+ }
+ }
+ (parent.tasks.find() {it.name=='getImageManifest'} ?: parent.task('getImageManifest'))\
+ .dependsOn getImageManifest
+ } else {
+ task dockerPushImage() { warnNoRegistryCredentials() }
+ task getImageManifest() { warnNoRegistryCredentials() }
+ }
+}
+
+/*
+ And now, the Manifest List. It's a multi-architecture manifest, so there's
+ no point to putting it in the individual architecture builds. Instead, it
+ will collect the individual manifests from the getManifest steps.
+ */
+configure (buildProjects) {
+ if (gradle?.registryCredentials) {
+ task putManifestList(type: PutManifestList, dependsOn: 'getImageManifest') {
+
+ targetRegistry = 'default'
+
+ imageName = "${findProperty('dockerImagePrefix') ?: 'openwhisk'}/" +
+ "${project.dockerImageName}"
+ // TODO - Project property candidate to support release builds?
+ tag = "${findProperty('dockerImageTag') ?: 'latest' }"
+
+ /*
+ Collect the manifests to be built from the subprojects
+ */
+ project.subprojects.collectMany { subproject ->
+ subproject.tasks.withType(GetImageManifest)
+ }.each { task ->
+ logger.info "Adding manifest ${it.path} to Manifest List"
+ manifests << { task.manifestMap }
+ }
+ }
+ } else {
+ task putManifestList() { warnNoRegistryCredentials() }
}
}
diff --git a/core/nodejs6Action/Dockerfile b/core/nodejs6Action/Dockerfile
index 102af6d..7d440bf 100644
--- a/core/nodejs6Action/Dockerfile
+++ b/core/nodejs6Action/Dockerfile
@@ -2,10 +2,10 @@ FROM nodejsactionbase
# based on https://github.com/nodejs/docker-node
ENV NODE_VERSION 6.12.2
-RUN curl -SLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-x64.tar.gz" \
- && tar -xzf "node-v$NODE_VERSION-linux-x64.tar.gz" -C /usr/local --strip-components=1 \
- && rm "node-v$NODE_VERSION-linux-x64.tar.gz"
-
+RUN node_arch=$(uname -m | sed -e 's/x86_64/x64/g' -e 's/aarch64/arm64/g') \
+ && curl -sSLO "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-${node_arch}.tar.gz" \
+ && tar -xzf "node-v$NODE_VERSION-linux-${node_arch}.tar.gz" -C /usr/local --strip-components=1 \
+ && rm "node-v$NODE_VERSION-linux-${node_arch}.tar.gz"
# workaround for this: https://github.com/npm/npm/issues/9863
RUN cd $(npm root -g)/npm \
diff --git a/core/nodejs6Action/build.gradle b/core/nodejs6Action/build.gradle
index b271f5e..b52d390 100644
--- a/core/nodejs6Action/build.gradle
+++ b/core/nodejs6Action/build.gradle
@@ -6,6 +6,12 @@ eclipse {
}
}
-ext.dockerImageName = 'nodejs6action'
-apply from: '../../gradle/docker.gradle'
-distDocker.dependsOn ':core:nodejsActionBase:distDocker'
+//ext.dockerImageName = 'nodejs6action'
+
+subprojects {
+ dockerBuildImage.with() {
+ dependsOn ':core:nodejsActionBase:dockerBuildImage'
+ inputDir=projectDir
+ }
+}
+
diff --git a/core/nodejs8Action/.dockerignore b/core/nodejs8Action/.dockerignore
index 3081e3e..3c8dcff 100644
--- a/core/nodejs8Action/.dockerignore
+++ b/core/nodejs8Action/.dockerignore
@@ -1,6 +1,5 @@
node_modules
package-lock.json
-Dockerfile
build.gradle
.project
.settings
diff --git a/core/nodejs8Action/build.gradle b/core/nodejs8Action/build.gradle
index f796acb..c32b335 100644
--- a/core/nodejs8Action/build.gradle
+++ b/core/nodejs8Action/build.gradle
@@ -6,13 +6,8 @@ eclipse {
}
}
-ext.dockerImageName = 'action-nodejs-v8'
-apply from: '../../gradle/docker.gradle'
-
-distDocker.dependsOn 'copyProxy'
-distDocker.dependsOn 'copyRunner'
-distDocker.dependsOn 'copyService'
-distDocker.finalizedBy('cleanup')
+//ext.dockerImageName = 'action-nodejs-v8'
+//apply from: '../../gradle/docker.gradle'
task copyProxy(type: Copy) {
from '../nodejsActionBase/app.js'
@@ -33,4 +28,14 @@ task cleanup(type: Delete) {
delete 'app.js'
delete 'runner.js'
delete 'src'
-}
\ No newline at end of file
+}
+
+subprojects {
+ dockerBuildImage.with() {
+ dependsOn copyProxy
+ dependsOn copyRunner
+ dependsOn copyService
+ inputDir=projectDir
+ finalizedBy(cleanup)
+ }
+}
diff --git a/core/nodejsActionBase/.dockerignore b/core/nodejsActionBase/.dockerignore
deleted file mode 100644
index 6df164e..0000000
--- a/core/nodejsActionBase/.dockerignore
+++ /dev/null
@@ -1,2 +0,0 @@
-Dockerfile
-build.gradle
diff --git a/core/nodejsActionBase/Dockerfile b/core/nodejsActionBase/Dockerfile.in
similarity index 58%
rename from core/nodejsActionBase/Dockerfile
rename to core/nodejsActionBase/Dockerfile.in
index 65de3d4..e7fe847 100644
--- a/core/nodejsActionBase/Dockerfile
+++ b/core/nodejsActionBase/Dockerfile.in
@@ -1,4 +1,5 @@
-FROM buildpack-deps:trusty-curl
+<% dockerInfo = dockerInfoClosure.call() %>
+FROM buildpack-deps:${(dockerInfo.architecture=='s390x')?'xenial':'trusty'}-curl
ENV DEBIAN_FRONTEND noninteractive
diff --git a/core/nodejsActionBase/build.gradle b/core/nodejsActionBase/build.gradle
index f6ae7ee..b5ad9a1 100644
--- a/core/nodejsActionBase/build.gradle
+++ b/core/nodejsActionBase/build.gradle
@@ -6,5 +6,26 @@ eclipse {
}
}
+import com.bmuschko.gradle.docker.tasks.DockerInfo
+
+subprojects {
+ task dockerFiles(type: Copy, dependsOn: dockerInfo) {
+
+ from('.') {
+ include '*.js'
+ include 'package.json'
+ }
+ from('./Dockerfile.in') {
+ rename { 'Dockerfile' }
+ expand( dockerInfoClosure: { project.dockerInfo.dockerInfo } )
+ }
+ into new File(buildDir, "docker")
+ }
+
+ dockerBuildImage.with() {
+ dependsOn dockerFiles
+ inputDir=new File(buildDir, "docker")
+ }
+}
+
ext.dockerImageName = 'nodejsactionbase'
-apply from: '../../gradle/docker.gradle'
diff --git a/docker-local.gradle.sample b/docker-local.gradle.sample
new file mode 100644
index 0000000..e366d00
--- /dev/null
+++ b/docker-local.gradle.sample
@@ -0,0 +1,27 @@
+//
+// If you intend to connect to a docker registry, either to push an image
+// or to get and put manifests, you need to provide registry credentials.
+//
+
+// TODO - maybe we should just default to these environment variables
+// and/or properties and skip using the file for this info?
+
+gradle.ext.dockerRegistry = env['DOCKER_REGISTRY'] // e.g. 'registry.docker.com'
+gradle.ext.registryCredentials = [
+ name: gradle.ext.dockerRegistry,
+ url: "https://${gradle.ext.dockerRegistry}/v2/" as String,
+ username: env['DOCKER_REGISTRY_USERNAME'],
+ password: env['DOCKER_REGISTRY_PASSWORD']
+ email: env['DOCKER_REGISTRY_EMAIL']
+]
+
+//
+// For multi-architecture builds:
+//
+gradle.ext.architectures = [
+ amd64: null,
+ s390x: [
+ url: 'https://sample.s390x.com:2376',
+ certPath: new File(System.properties['user.home'], "./.tls/s390x")
+ ]
+]
diff --git a/gradle/docker.gradle b/gradle/docker.gradle
deleted file mode 100644
index f716c7b..0000000
--- a/gradle/docker.gradle
+++ /dev/null
@@ -1,99 +0,0 @@
-import groovy.time.*
-
-/**
- * Utility to build docker images based in gradle projects
- *
- * This extends gradle's 'application' plugin logic with a 'distDocker' task which builds
- * a docker image from the Dockerfile of the project that applies this file. The image
- * is automatically tagged and pushed if a tag and/or a registry is given.
- *
- * Parameters that can be set on project level:
- * - dockerImageName (required): The name of the image to build (e.g. controller)
- * - dockerRegistry (optional): The registry to push to
- * - dockerImageTag (optional, default 'latest'): The tag for the image
- * - dockerImagePrefix (optional, default 'whisk'): The prefix for the image,
- * 'controller' becomes 'whisk/controller' per default
- * - dockerTimeout (optional, default 840): Timeout for docker operations in seconds
- * - dockerRetries (optional, default 3): How many times to retry docker operations
- * - dockerBinary (optional, default 'docker'): The binary to execute docker commands
- * - dockerBuildArgs (options, default ''): Project specific custom docker build arguments
- * - dockerHost (optional): The docker host to run commands on, default behaviour is
- * docker's own DOCKER_HOST environment variable
- */
-
-ext {
- dockerRegistry = project.hasProperty('dockerRegistry') ? dockerRegistry + '/' : ''
- dockerImageTag = project.hasProperty('dockerImageTag') ? dockerImageTag : 'latest'
- dockerImagePrefix = project.hasProperty('dockerImagePrefix') ? dockerImagePrefix : 'whisk'
- dockerTimeout = project.hasProperty('dockerTimeout') ? dockerTimeout.toInteger() : 840
- dockerRetries = project.hasProperty('dockerRetries') ? dockerRetries.toInteger() : 3
- dockerBinary = project.hasProperty('dockerBinary') ? [dockerBinary] : ['docker']
- dockerBuildArg = ['build']
-}
-ext.dockerTaggedImageName = dockerRegistry + dockerImagePrefix + '/' + dockerImageName + ':' + dockerImageTag
-
-if(project.hasProperty('dockerHost')) {
- dockerBinary += ['--host', project.dockerHost]
-}
-
-if(project.hasProperty('dockerBuildArgs')) {
- dockerBuildArgs.each { arg ->
- dockerBuildArg += ['--build-arg', arg]
- }
-}
-
-task distDocker {
- doLast {
- def start = new Date()
- def cmd = dockerBinary + dockerBuildArg + ['-t', dockerImageName, project.buildscript.sourceFile.getParentFile().getAbsolutePath()]
- retry(cmd, dockerRetries, dockerTimeout)
- println("Building '${dockerImageName}' took ${TimeCategory.minus(new Date(), start)}")
- }
-}
-task tagImage {
- doLast {
- def versionString = (dockerBinary + ['-v']).execute().text
- def matched = (versionString =~ /(\d+)\.(\d+)\.(\d+)/)
-
- def major = matched[0][1] as int
- def minor = matched[0][2] as int
-
- def dockerCmd = ['tag']
- if(major == 1 && minor < 12) {
- dockerCmd += ['-f']
- }
- retry(dockerBinary + dockerCmd + [dockerImageName, dockerTaggedImageName], dockerRetries, dockerTimeout)
- }
-}
-
-task pushImage {
- doLast {
- def cmd = dockerBinary + ['push', dockerTaggedImageName]
- retry(cmd, dockerRetries, dockerTimeout)
- }
-}
-pushImage.dependsOn tagImage
-pushImage.onlyIf { dockerRegistry != '' }
-distDocker.finalizedBy pushImage
-
-def retry(cmd, retries, timeout) {
- println("${new Date()}: Executing '${cmd.join(" ")}'")
- def proc = cmd.execute()
- proc.consumeProcessOutput(System.out, System.err)
- proc.waitForOrKill(timeout * 1000)
- if(proc.exitValue() != 0) {
- def message = "${new Date()}: Command '${cmd.join(" ")}' failed with exitCode ${proc.exitValue()}"
- if(proc.exitValue() == 143) { // 143 means the process was killed (SIGTERM signal)
- message = "${new Date()}: Command '${cmd.join(" ")}' was killed after ${timeout} seconds"
- }
-
- if(retries > 1) {
- println("${message}, ${retries-1} retries left, retrying...")
- retry(cmd, retries-1, timeout)
- }
- else {
- println("${message}, no more retries left, aborting...")
- throw new GradleException(message)
- }
- }
-}
diff --git a/settings.gradle b/settings.gradle
index c304cf9..90f900f 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -1,9 +1,42 @@
include 'tests'
-include 'core:nodejsActionBase'
-include 'core:nodejs6Action'
+// TODO: Can this file be specified as a parameter?
+file('./docker-local.gradle').with() {
+ if (it.exists()) {
+ apply from: it
+ }
+}
-include 'core:nodejs8Action'
+if (!gradle.ext.has('architectures')) gradle.ext.architectures = [ default: [:] ]
+
+gradle.ext.dockerBuildProjects =
+[
+ ':core:nodejsActionBase': [
+ 'dockerImageName': 'nodejsactionbase'
+ ],
+ ':core:nodejs6Action': [
+ 'dockerImageName': 'nodejs6action'
+ ],
+ ':core:nodejs8Action': [
+ 'dockerImageName': 'action-nodejs-v8'
+ ]
+]
+
+gradle.dockerBuildProjects.each() { baseProjectName, extensions ->
+ include baseProjectName
+ def baseProject = findProject(baseProjectName)
+
+ gradle.architectures.each() { architectureName, architectureClosure ->
+ def architectureProjectName = "${baseProjectName}:${architectureName}"
+ include architectureProjectName
+ def architectureProject = findProject(architectureProjectName)
+
+ logger.debug 'Setup baseProject: ' + baseProject?.path
+ logger.debug 'Setup architectureProject: ' + architectureProject?.path
+
+ architectureProject.projectDir = baseProject.projectDir
+ }
+}
rootProject.name = 'runtime-nodejs'
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index 19ea582..efb3a98 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -1,21 +1,4 @@
#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
set -ex
# Build script for Travis-CI.
@@ -36,8 +19,8 @@ docker pull openwhisk/controller
docker tag openwhisk/controller ${IMAGE_PREFIX}/controller
docker pull openwhisk/invoker
docker tag openwhisk/invoker ${IMAGE_PREFIX}/invoker
-docker pull openwhisk/nodejs6action
-docker tag openwhisk/nodejs6action ${IMAGE_PREFIX}/nodejs6action
+#docker pull openwhisk/nodejs6action
+#docker tag openwhisk/nodejs6action ${IMAGE_PREFIX}/nodejs6action
TERM=dumb ./gradlew \
:common:scala:install \
@@ -48,6 +31,6 @@ TERM=dumb ./gradlew \
# Build runtime
cd $ROOTDIR
TERM=dumb ./gradlew \
-:core:nodejs6Action:distDocker \
-:core:nodejs8Action:distDocker \
+:core:nodejs6Action:dockerBuildImage \
+:core:nodejs8Action:dockerBuildImage \
-PdockerImagePrefix=${IMAGE_PREFIX}
diff --git a/tools/travis/docker-local.gradle.gpg b/tools/travis/docker-local.gradle.gpg
new file mode 100644
index 0000000..5a5f369
Binary files /dev/null and b/tools/travis/docker-local.gradle.gpg differ
diff --git a/tools/travis/docker-local.gradle.template b/tools/travis/docker-local.gradle.template
new file mode 100644
index 0000000..7ae9b27
--- /dev/null
+++ b/tools/travis/docker-local.gradle.template
@@ -0,0 +1,34 @@
+// There are intended to be accessed within settings.gradle
+// to configure the build
+
+/*
+ This file provides the guidelines for the file (docker-local.gradle.gpg)
+ that is to be encoded and provided to Travis to execute tests.
+
+ TODO: Once the basics are working, allow configuration of the file to
+ be used based on the current repository. Or maybe use a JSON-encoded
+ environment variable? Which is even more complicated...
+ */
+
+// The key serves as a unique name for the rest of the processing.
+gradle.ext.architectures = [
+ amd64: null,
+ ppc64le: [
+ url: 'https://<ip_address>:2376',
+ certPath: rootProject.file("tools/travis/tls/ppc64le")
+ ],
+ s390x: [
+ url: 'https://<ip_address>:2376',
+ certPath: rootProject.file("tools/travis/tls/s390x")
+ ]
+]
+
+gradle.ext.dockerRegistry = '<registry_dns_name>'
+
+gradle.ext.registryCredentials = [
+ name: gradle.ext.dockerRegistry,
+ url: "https://${gradle.ext.dockerRegistry}/v2/" as String,
+ username: '<registry_username>',
+ password: '<registry_password>',
+ email: '<registry_email>'
+]
diff --git a/tools/travis/publish.sh b/tools/travis/publish.sh
index cfdd8d3..f49b4de 100755
--- a/tools/travis/publish.sh
+++ b/tools/travis/publish.sh
@@ -36,14 +36,9 @@ elif [ ${RUNTIME_VERSION} == "8" ]; then
RUNTIME="nodejs8Action"
fi
-if [[ ! -z ${DOCKER_USER} ]] && [[ ! -z ${DOCKER_PASSWORD} ]]; then
-docker login -u "${DOCKER_USER}" -p "${DOCKER_PASSWORD}"
-fi
-
if [[ ! -z ${RUNTIME} ]]; then
TERM=dumb ./gradlew \
-:core:${RUNTIME}:distDocker \
--PdockerRegistry=docker.io \
+:core:${RUNTIME}:pubManifestList \
-PdockerImagePrefix=${IMAGE_PREFIX} \
-PdockerImageTag=${IMAGE_TAG}
fi
diff --git a/tools/travis/tls/ppc64le/ca.pem.gpg b/tools/travis/tls/ppc64le/ca.pem.gpg
new file mode 100644
index 0000000..2d53007
Binary files /dev/null and b/tools/travis/tls/ppc64le/ca.pem.gpg differ
diff --git a/tools/travis/tls/ppc64le/cert.pem.gpg b/tools/travis/tls/ppc64le/cert.pem.gpg
new file mode 100644
index 0000000..1ff14aa
Binary files /dev/null and b/tools/travis/tls/ppc64le/cert.pem.gpg differ
diff --git a/tools/travis/tls/ppc64le/key.pem.gpg b/tools/travis/tls/ppc64le/key.pem.gpg
new file mode 100644
index 0000000..7b898a8
Binary files /dev/null and b/tools/travis/tls/ppc64le/key.pem.gpg differ
diff --git a/tools/travis/tls/s390x/ca.pem.gpg b/tools/travis/tls/s390x/ca.pem.gpg
new file mode 100644
index 0000000..69776e6
Binary files /dev/null and b/tools/travis/tls/s390x/ca.pem.gpg differ
diff --git a/tools/travis/tls/s390x/cert.pem.gpg b/tools/travis/tls/s390x/cert.pem.gpg
new file mode 100644
index 0000000..8910f66
Binary files /dev/null and b/tools/travis/tls/s390x/cert.pem.gpg differ
diff --git a/tools/travis/tls/s390x/key.pem.gpg b/tools/travis/tls/s390x/key.pem.gpg
new file mode 100644
index 0000000..c8e11f6
Binary files /dev/null and b/tools/travis/tls/s390x/key.pem.gpg differ
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services