You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2017/12/23 00:04:23 UTC
[GitHub] jonpspri closed pull request #3108: S390x execution of OpenWhisk
jonpspri closed pull request #3108: S390x execution of OpenWhisk
URL: https://github.com/apache/incubator-openwhisk/pull/3108
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index bfb7f48f58..cf1556afa4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -15,7 +15,7 @@ results
!/ansible/environments/distributed
!/ansible/environments/docker-machine
!/ansible/environments/local
-!/ansible/environments/mac
+!/ansible/environments/s390x_local
# Eclipse
bin/
@@ -30,6 +30,7 @@ bin/
# Mac
.DS_Store
+._*
# Gradle
.gradle
@@ -45,6 +46,7 @@ node_modules
# Vagrant
.vagrant*
+tools/vagrant/*.log
# IntelliJ
.idea
diff --git a/ansible/environments/s390x_local/group_vars/all b/ansible/environments/s390x_local/group_vars/all
new file mode 100755
index 0000000000..d4638497ba
--- /dev/null
+++ b/ansible/environments/s390x_local/group_vars/all
@@ -0,0 +1,56 @@
+whisk_version_name: local
+config_root_dir: /tmp/wskconf
+whisk_logs_dir: /tmp/wsklogs
+docker_registry: ""
+docker_dns: ""
+
+db_prefix: "{{ ansible_user_id|lower }}_{{ ansible_hostname|lower }}_"
+
+# Auto lookup to find the db credentials
+db_provider: "{{ lookup('ini', 'db_provider section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_username: "{{ lookup('ini', 'db_username section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_password: "{{ lookup('ini', 'db_password section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_protocol: "{{ lookup('ini', 'db_protocol section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_host: "{{ lookup('ini', 'db_host section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+db_port: "{{ lookup('ini', 'db_port section=db_creds file={{ playbook_dir }}/db_local.ini') }}"
+
+# API GW connection configuration
+apigw_auth_user: ""
+apigw_auth_pwd: ""
+apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2"
+
+controller_arguments: '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098'
+invoker_arguments: "{{ controller_arguments }}"
+
+invoker_allow_multiple_instances: true
+
+# Set kafka configuration
+kafka_heap: '512m'
+kafka_topics_completed_retentionBytes: 104857600
+kafka_topics_completed_retentionMS: 300000
+kafka_topics_health_retentionBytes: 104857600
+kafka_topics_health_retentionMS: 300000
+kafka_topics_invoker_retentionBytes: 104857600
+kafka_topics_invoker_retentionMS: 300000
+
+# RunC enablement
+invoker_use_runc: true
+invoker_use_reactive_pool: false
+
+### S390X specifics (TODO: could I include what's above?) ###
+
+nginx_version: 1.12
+
+# Docker images to pull for S390X docker
+
+couchdb:
+ docker_image: "docker.xanophis.com/s390x/couchdb:latest"
+
+kafka:
+ docker_image: "docker.xanophis.com/s390x/wurstmeister-kafka:0.11"
+
+apigateway:
+ docker_image: "docker.xanophis.com/s390x/openwhisk-apigateway:latest"
+
+# Because we don't (yet) support s390x compiles of the CLI; a different challenge...
+cli_installation_mode: local
diff --git a/ansible/environments/s390x_local/hosts b/ansible/environments/s390x_local/hosts
new file mode 100644
index 0000000000..4cf54aa811
--- /dev/null
+++ b/ansible/environments/s390x_local/hosts
@@ -0,0 +1,31 @@
+; the first parameter in a host is the inventory_hostname
+
+; used for local actions only
+ansible ansible_connection=local
+
+[edge]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
+
+[controllers]
+controller0 ansible_host=172.17.0.1 ansible_connection=local
+
+[kafkas]
+kafka0 ansible_host=172.17.0.1 ansible_connection=local
+kafka1 ansible_host=172.17.0.1 ansible_connection=local
+
+[zookeepers:children]
+kafkas
+
+[invokers]
+invoker0 ansible_host=172.17.0.1 ansible_connection=local
+invoker1 ansible_host=172.17.0.1 ansible_connection=local
+
+; db group is only used if db_provider is CouchDB
+[db]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
+
+[redis]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
+
+[apigateway]
+172.17.0.1 ansible_host=172.17.0.1 ansible_connection=local
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index 5cde947173..61df6b949f 100644
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -179,7 +179,7 @@ nginx:
confdir: "{{ config_root_dir }}/nginx"
dir:
become: "{{ nginx_dir_become | default(false) }}"
- version: 1.11
+ version: "{{ nginx_version | default(1.11) }}"
port:
http: 80
api: 443
diff --git a/ansible/roles/cli/tasks/deploy.yml b/ansible/roles/cli/tasks/deploy.yml
index 3961ab84c0..3554ea6972 100644
--- a/ansible/roles/cli/tasks/deploy.yml
+++ b/ansible/roles/cli/tasks/deploy.yml
@@ -26,3 +26,4 @@
when: cli_installation_mode == "local"
- include: download_cli.yml
+ when: cli_installation_mode == "remote"
diff --git a/ansible/roles/couchdb/tasks/deploy.yml b/ansible/roles/couchdb/tasks/deploy.yml
index 542111a1cb..675a8ca372 100644
--- a/ansible/roles/couchdb/tasks/deploy.yml
+++ b/ansible/roles/couchdb/tasks/deploy.yml
@@ -21,12 +21,16 @@
volume_dir: "{{ instance.volume.fsmount | default( '/mnt/' + group_names|first, true ) }}:/usr/local/var/lib/couchdb"
when: (block_device is defined) and (block_device in disk_status.stdout)
-- name: "pull the apache/couchdb:{{ couchdb.version }} image"
- shell: "docker pull apache/couchdb:{{ couchdb.version }}"
+- name: "determine the docker_image to use"
+ set_fact:
+ couch_image: "{{ couchdb.docker_image | default('apache/couchdb:' ~ couchdb.version ) }}"
+
+- name: "pull the {{ couch_image }} image"
+ shell: "docker pull {{ couch_image }}"
retries: "{{ docker.pull.retries }}"
delay: "{{ docker.pull.delay }}"
-- name: (re)start CouchDB
+- name: "(re)start CouchDB"
docker_container:
name: couchdb
image: apache/couchdb:{{ couchdb.version }}
diff --git a/ansible/roles/kafka/tasks/deploy.yml b/ansible/roles/kafka/tasks/deploy.yml
index 7c6fedae95..f0278df123 100644
--- a/ansible/roles/kafka/tasks/deploy.yml
+++ b/ansible/roles/kafka/tasks/deploy.yml
@@ -1,8 +1,12 @@
---
# This role will install Kafka with Zookeeper in group 'kafka' in the environment inventory
-- name: "pull the wurstmeister/kafka:{{ kafka.version }} image"
- shell: "docker pull wurstmeister/kafka:{{ kafka.version }}"
+- name: Set the kafka docker image names
+ set_fact:
+ kafka_image: "{{ kafka.docker_image | default ('wurstmeister/kafka:' ~ kafka.version) }}"
+
+- name: "pull the {{ kafka_image }} image"
+ shell: "docker pull {{ kafka_image }}"
retries: "{{ docker.pull.retries }}"
delay: "{{ docker.pull.delay }}"
@@ -11,7 +15,7 @@
zookeeper_idx: "{{ groups['kafkas'].index(inventory_hostname) % (groups['zookeepers'] | length) }}"
docker_container:
name: kafka{{ groups['kafkas'].index(inventory_hostname) }}
- image: wurstmeister/kafka:{{ kafka.version }}
+ image: "{{ kafka_image }}"
state: started
recreate: true
restart_policy: "{{ docker.restart.policy }}"
diff --git a/common/scala/Dockerfile.s390x b/common/scala/Dockerfile.s390x
new file mode 100644
index 0000000000..2ba228065c
--- /dev/null
+++ b/common/scala/Dockerfile.s390x
@@ -0,0 +1,9 @@
+FROM buildpack-deps:xenial-curl
+
+RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y \
+ openjdk-8-jre-headless \
+ wget \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY transformEnvironment.sh /
+RUN chmod +x transformEnvironment.sh
diff --git a/core/invoker/Dockerfile.s390x b/core/invoker/Dockerfile.s390x
new file mode 100644
index 0000000000..09eab34123
--- /dev/null
+++ b/core/invoker/Dockerfile.s390x
@@ -0,0 +1,15 @@
+FROM scala
+
+RUN apt-get update \
+ && DEBIAN_FRONTEND="noninteractive" apt-get install -y docker.io \
+ && rm -rf /var/lib/apt/lists/*
+
+COPY build/distributions/invoker.tar ./
+RUN tar xf invoker.tar && \
+rm -f invoker.tar
+
+COPY init.sh /
+RUN chmod +x init.sh
+
+EXPOSE 8080
+CMD ["init.sh", "0"]
\ No newline at end of file
diff --git a/gradle/docker.gradle b/gradle/docker.gradle
index f716c7b400..9a226b0d8e 100644
--- a/gradle/docker.gradle
+++ b/gradle/docker.gradle
@@ -19,8 +19,23 @@ import groovy.time.*
* - dockerBuildArgs (options, default ''): Project specific custom docker build arguments
* - dockerHost (optional): The docker host to run commands on, default behaviour is
* docker's own DOCKER_HOST environment variable
+ * - dockerArch (optional): An alternate architecture to generate in Docker. The script
+ * will expect to see a Dockerfile.dockerArch (e.g. Dockerfile.s390x) or a patch
+ * file to be applied to the Dockerfile to create the new architecture.
*/
+def detectDockerArch() {
+ def stdout = new ByteArrayOutputStream()
+ exec {
+ executable '/bin/bash'
+ args '-c', "docker info 2>/dev/null | sed -n 's/Architecture: \\(.*\\)/\\1/p'"
+ standardOutput = stdout;
+ }
+ def arch = stdout.toString().trim()
+ // println "Docker architecture ${arch} detected"
+ return arch
+}
+
ext {
dockerRegistry = project.hasProperty('dockerRegistry') ? dockerRegistry + '/' : ''
dockerImageTag = project.hasProperty('dockerImageTag') ? dockerImageTag : 'latest'
@@ -29,6 +44,7 @@ ext {
dockerRetries = project.hasProperty('dockerRetries') ? dockerRetries.toInteger() : 3
dockerBinary = project.hasProperty('dockerBinary') ? [dockerBinary] : ['docker']
dockerBuildArg = ['build']
+ dockerArch = project.hasProperty('dockerArch') ? dockerArch : detectDockerArch()
}
ext.dockerTaggedImageName = dockerRegistry + dockerImagePrefix + '/' + dockerImageName + ':' + dockerImageTag
@@ -42,14 +58,42 @@ if(project.hasProperty('dockerBuildArgs')) {
}
}
+task templateDockerfile (type: Copy) {
+ def directory = project.buildscript.sourceFile.getParentFile()
+ from directory
+ into directory
+ include 'Dockerfile.in'
+ rename { file -> 'Dockerfile' }
+ expand(project.getProperties())
+}
+
+task rmDockerfile(type: Delete) {
+ delete 'Dockerfile'
+}
+
task distDocker {
doLast {
def start = new Date()
- def cmd = dockerBinary + dockerBuildArg + ['-t', dockerImageName, project.buildscript.sourceFile.getParentFile().getAbsolutePath()]
+ def directory = project.buildscript.sourceFile.getParentFile()
+ def cmd = dockerBinary + dockerBuildArg
+ if (dockerArch) {
+ // See whether there's an architecture-specific Dockerfile in place
+ def f = new File(directory.getAbsolutePath(), "Dockerfile."+dockerArch);
+ if (f.exists()) {
+ cmd += ['-f', f.getAbsolutePath()]
+ }
+ }
+ cmd += ['-t', dockerImageName, directory.getAbsolutePath()]
retry(cmd, dockerRetries, dockerTimeout)
println("Building '${dockerImageName}' took ${TimeCategory.minus(new Date(), start)}")
}
}
+
+if (file("Dockerfile.in").exists()) {
+ distDocker.dependsOn 'templateDockerfile'
+ distDocker.finalizedBy 'rmDockerfile'
+}
+
task tagImage {
doLast {
def versionString = (dockerBinary + ['-v']).execute().text
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services