You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/02/28 23:43:11 UTC
[GitHub] jonpspri closed pull request #3216: Download Nginx Tarball from OpenWhisk CLI release, rather than constructing file-by-file
jonpspri closed pull request #3216: Download Nginx Tarball from OpenWhisk CLI release, rather than constructing file-by-file
URL: https://github.com/apache/incubator-openwhisk/pull/3216
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ansible/edge.yml b/ansible/edge.yml
index 6566d572ae..f2733a0f2f 100644
--- a/ansible/edge.yml
+++ b/ansible/edge.yml
@@ -1,8 +1,8 @@
---
# This playbook deploys Openwhisk Edge servers.
# The edge is usually populated with NGINX serving as proxy.
-# The CLI also gets built and published for downloading from NGINX.
-# SDKs for blackbox and iOS get published to NGINX also.
+# The CLI also gets built and published for downloading from NGINX.
+# SDKs for blackbox and iOS get published to NGINX also ( )
- hosts: edge
roles:
diff --git a/ansible/group_vars/all b/ansible/group_vars/all
index ce2054352d..2ec8d641f1 100644
--- a/ansible/group_vars/all
+++ b/ansible/group_vars/all
@@ -1,7 +1,8 @@
mode: deploy
prompt_user: true
-openwhisk_home: "{{ lookup('env', 'OPENWHISK_HOME') | default(playbook_dir + '/..', true) }}"
+openwhisk_home: "{{ lookup('env', 'OPENWHISK_HOME') | default(playbook_dir ~ '/..', true) }}"
openwhisk_cli_home: "{{ lookup('env', 'OPENWHISK_CLI') | default(openwhisk_home ~ '/../incubator-openwhisk-cli', true) }}"
+openwhisk_build_dir: "{{ openwhisk_home }}/build"
exclude_logs_from: []
# This whisk_api_localhost_name_default is used to configure nginx to permit vanity URLs for web actions
@@ -24,9 +25,17 @@ whisk:
version:
date: "{{ansible_date_time.iso8601}}"
-
-# list of supported runtimes (see whisk.core.entity.ExecManifest for schema)
-runtimesManifest: "{{ runtimes_manifest | default(lookup('file', '{{ openwhisk_home }}/ansible/files/runtimes.json') | from_json) }}"
+##
+# list of supported runtimes (see whisk.core.entity.ExecManifest for schema).
+# briefly:
+# defaultImagePrefix: the default image prefix when not given explicitly
+# defaultImageTag: the default image tag
+# runtimes: set of language runtime families grouped by language (e.g., nodejs, python)
+# blackboxes: list of pre-populated docker action images as "name" with optional "prefix" and "tag"
+# bypassPullForLocalImages: optional, if true, allow images with a prefix that matches {{ docker.image.prefix }}
+# to skip docker pull in invoker even if the image is not part of the blackbox set
+#
+runtimesManifest: "{{ runtimes_manifest | default(lookup('file', openwhisk_home ~ '/ansible/files/runtimes.json') | from_json) }}"
limits:
invocationsPerMinute: "{{ limit_invocations_per_minute | default(60) }}"
@@ -213,7 +222,6 @@ sdk:
cli:
path: "{{ openwhisk_home }}/bin/wsk"
- nginxdir: "{{ nginx.confdir }}/cli/go/download"
dir:
become: "{{ cli_dir_become | default(false) }}"
@@ -254,8 +262,6 @@ catalog_repos:
version: "HEAD"
repo_update: "no"
-openwhisk_cli_tag: "{{ lookup('ini', 'git_tag section=openwhisk-cli file={{ openwhisk_home }}/ansible/files/package-versions.ini') }}"
-
# The openwhisk_cli is used to determine how to install the OpenWhisk CLI. The
# installation_mode can be specified into two modes: remote and local.
# The mode remote means to download the available binaries from the releases page
@@ -270,26 +276,18 @@ openwhisk_cli_tag: "{{ lookup('ini', 'git_tag section=openwhisk-cli file={{ open
# The location specifies the official website where Openwhisk CLI is hosted in
# remote mode or location to save the binaries of the OpenWhisk CLI in local mode.
+openwhisk_cli_tag: "{{ cli_tag | default(lookup('ini', 'git_tag section=openwhisk-cli file=' ~ openwhisk_home ~ '/ansible/files/package-versions.ini')) }}"
openwhisk_cli:
installation_mode: "{{ cli_installation_mode | default(lookup('env', 'OPENWHISK_CLI_MODE')) | default('remote', true) }}"
+ archive_name: OpenWhisk_CLI
+ nginxdir:
+ name: "{{ nginx.confdir }}/cli/go/download"
+ become: "{{ cli_dir_become | default(false) }}"
local:
location: "{{ openwhisk_cli_home }}/build"
remote:
- name: OpenWhisk_CLI
- dest_name: OpenWhisk_CLI
location: "https://github.com/apache/incubator-openwhisk-cli/releases/download/{{ openwhisk_cli_tag }}"
-# The list of operating systems and architectures for which openwhisk cli binaries are downloaded,
-# if the installation_mode is remote.
-cli_os_arch:
- - [ "linux", "amd64" ]
- - [ "linux", "386" ]
- - [ "linux", "s390x" ]
- - [ "mac", "amd64" ]
- - [ "mac", "386" ]
- - [ "windows", "amd64" ]
- - [ "windows", "386" ]
-
# Controls access to log directories
logs:
dir:
diff --git a/ansible/roles/cli-install/tasks/clean.yml b/ansible/roles/cli-install/tasks/clean.yml
new file mode 100644
index 0000000000..b3eae8f711
--- /dev/null
+++ b/ansible/roles/cli-install/tasks/clean.yml
@@ -0,0 +1,8 @@
+---
+
+- name: remove cli nginx directory
+ file:
+ path: "{{ openwhisk_home }}/bin/{{ wsk }}"
+ state: absent
+ vars:
+ wsk: "{{ ( ansible_system == 'Windows') | ternary('wsk.exe', 'wsk') }}"
diff --git a/ansible/roles/cli-install/tasks/deploy.yml b/ansible/roles/cli-install/tasks/deploy.yml
new file mode 100644
index 0000000000..b2a30ccfe8
--- /dev/null
+++ b/ansible/roles/cli-install/tasks/deploy.yml
@@ -0,0 +1,14 @@
+---
+# Install the appropriate CLI into the ansible host for the routemgmt tasks
+
+- name: "Grab the local CLI from the binaries unarchived into nginx"
+ get_url:
+ url: "https://{{host}}/cli/go/download/{{os}}/{{arch}}/{{wsk}}"
+ dest: "{{ openwhisk_home }}/bin"
+ mode: "0755"
+ validate_certs: no
+ vars:
+ host: "{{ groups['edge'] | first }}"
+ arch: "{{ ansible_machine | replace ('x86_64', 'amd64') }}"
+ os: "{{ ansible_system | lower | replace('darwin', 'mac') }}"
+ wsk: "{{ ( ansible_system == 'Windows') | ternary('wsk.exe', 'wsk') }}"
diff --git a/ansible/roles/cli-install/tasks/main.yml b/ansible/roles/cli-install/tasks/main.yml
new file mode 100644
index 0000000000..b03b406c3a
--- /dev/null
+++ b/ansible/roles/cli-install/tasks/main.yml
@@ -0,0 +1,10 @@
+---
+# This role will configure and publish the CLI tarball.
+# In deploy mode it will generate a new CLI config, generate a tarball and copy it to nginx.
+# In clean mode it will clean the cli.nginxdir
+
+- include: deploy.yml
+ when: mode == "deploy"
+
+- include: clean.yml
+ when: mode == "clean"
diff --git a/ansible/roles/cli/files/createContentJson.sh b/ansible/roles/cli/files/createContentJson.sh
deleted file mode 100755
index 7942adf96f..0000000000
--- a/ansible/roles/cli/files/createContentJson.sh
+++ /dev/null
@@ -1,80 +0,0 @@
-#!/bin/bash
-
-set +x
-set -e
-
-get_compressed_name() {
- local os=$1
- local arch=$2
- local product_name=$3
-
- if [ $arch = amd64 ]; then
- comp_name="$product_name-$os";
- elif [ $arch = 386 ]; then
- comp_name="$product_name-$os-32bit";
- else
- comp_name="$product_name-$os-$arch";
- fi
-
- echo $comp_name;
-};
-
-get_binary_path() {
- local comp_name=$1
- local os=$2
- local arch=$3
-
- if [ $os = "linux" ]; then
- comp_name="$comp_name.tgz"
- else
- comp_name="$comp_name.zip"
- fi
- echo $os/$arch/$comp_name;
-};
-
-create_cli_packages() {
- local dirIndex="{\"cli\":{"
-
- # I'm making a nasty assumption here that it's sorted by platforms
- old_platform='' # Not really necessary, but here to be explicit & transparent
- for platform_arch in $platforms; do
-
- IFS="," read platform arch <<<"${platform_arch}"
-
- # Control-break processing for platform changes
- if [ "$platform" != "$old_platform" ]; then
- if [ "$old_platform" != "" ]; then
- dirIndex="${dirIndex%','}" # Remove trailing comma
- dirIndex+="}," # And replace with end-brace comma
- fi
- dirIndex+="\"$platform\":{"
- fi
- old_platform=$platform
-
- comp_name=$(get_compressed_name $platform $arch $zip_name)
- comp_path=$(get_binary_path $comp_name $platform $arch)
-
- if [ $arch = $default_arch ]; then
- dirIndex+="\"default\":{\"path\":\"$comp_path\"},";
- fi
-
- dirIndex+="\"$arch\":{\"path\":\"$comp_path\"},";
- done
-
- #dirIndex="$(echo $dirIndex | rev | cut -c2- | rev)"
- dirIndex="${dirIndex%','}" # Remove trailing comma
- dirIndex+="}}}" # And replace with end-braces
-
- mkdir -p "${PATH_CONTENT_JSON}"
- echo $dirIndex > "${PATH_CONTENT_JSON}/content.json"
-};
-
-default_arch="amd64"
-
-PATH_CONTENT_JSON=$1
-# 'shellcheck' pointed out that BINARY_TAG is unused. I'm leaving it for now
-# because the future of content.json is in flux.
-BINARY_TAG=$2
-platforms=$3
-zip_name=$4
-create_cli_packages
diff --git a/ansible/roles/cli/tasks/clean.yml b/ansible/roles/cli/tasks/clean.yml
index 6c365ab0cb..6899e132de 100644
--- a/ansible/roles/cli/tasks/clean.yml
+++ b/ansible/roles/cli/tasks/clean.yml
@@ -2,6 +2,6 @@
- name: remove cli nginx directory
file:
- path: "{{ cli.nginxdir }}"
+ path: "{{ openwhisk_cli.nginxdir.name }}"
state: absent
- become: "{{ cli.dir.become }}"
+ become: "{{ openwhisk_cli.nginxdir.become }}"
diff --git a/ansible/roles/cli/tasks/cli_remote_mode.yml b/ansible/roles/cli/tasks/cli_remote_mode.yml
deleted file mode 100644
index 7d16c8a109..0000000000
--- a/ansible/roles/cli/tasks/cli_remote_mode.yml
+++ /dev/null
@@ -1,24 +0,0 @@
----
-
-# Download the cli for the remote mode and generate the content.json file
-
-- name: "download the binaries from the release page of OpenWhisk CLI"
- include: download_openwhisk_cli.yml
- with_items:
- - "{{ cli_os_arch }}"
-
-- name: "generate content.json locally for Nginx CLI directory"
- local_action: shell "./createContentJson.sh" "{{ config_root_dir }}" "{{ openwhisk_cli_tag }}" "{{ cli_os_arch | sort | map('join', ',') | join(' ') }}" "{{ openwhisk_cli.remote.dest_name }}" chdir="{{ openwhisk_home }}/ansible/roles/cli/files"
-
-- name: "ensure Nginx cli directory is writable"
- file:
- path: "{{ cli.nginxdir }}"
- state: directory
- mode: 0777
- become: "{{ cli.dir.become }}"
-
-- name: "copy the local content.json to the remote node"
- copy: src={{ config_root_dir }}/content.json dest={{ cli.nginxdir }}/
-
-- name: "delete the local content.json"
- local_action: file path={{ config_root_dir }}/content.json state=absent
diff --git a/ansible/roles/cli/tasks/copy_local_openwhisk_cli.yml b/ansible/roles/cli/tasks/copy_local_openwhisk_cli.yml
deleted file mode 100644
index 18b0a44e61..0000000000
--- a/ansible/roles/cli/tasks/copy_local_openwhisk_cli.yml
+++ /dev/null
@@ -1,36 +0,0 @@
----
-
-# Copy the cli binaries to Nginx directory
-
-- name: "construct local file from which to source"
- set_fact:
- source_file: "{{ openwhisk_cli.local.location }}/\
- {{ item | join('-') | replace('mac','darwin') }}/\
- {{ (item[0] == 'windows') | ternary ('wsk.exe', 'wsk') }}"
- target_dir: "{{ cli.nginxdir }}/{{ item[0] }}/{{ item[1] }}"
- system_match: "{{ ( \
- (ansible_system == 'Linux' and item[0] == 'linux') \
- or (ansible_system == 'Darwin' and item[0] == 'mac') \
- or (ansible_system == 'Windows' and item[0] == 'windows') \
- ) }}"
- arch_match: "{{ ( ((ansible_machine=='x86_64')|ternary('amd64', ansible_machine)) == item[1] ) }}"
-
-- local_action: stat path={{ source_file }}
- register: binary_path
-
-- name: "ensure Nginx cli directory is writable"
- file:
- path: "{{ target_dir }}"
- state: directory
- mode: 0777
- become: "{{ cli.dir.become }}"
-
-- name: "copy the local binaries from a local directory to Nginx directory"
- copy:
- src: "{{ source_file }}"
- dest: "{{ target_dir }}/"
- when: binary_path.stat.exists
-
-- name: "copy the local binary to the root bin directory when architectures match"
- local_action: copy src={{ source_file }} dest={{ openwhisk_home }}/bin mode=0755
- when: system_match and arch_match
diff --git a/ansible/roles/cli/tasks/deploy.yml b/ansible/roles/cli/tasks/deploy.yml
index 20f622148d..8591263681 100644
--- a/ansible/roles/cli/tasks/deploy.yml
+++ b/ansible/roles/cli/tasks/deploy.yml
@@ -1,30 +1,88 @@
---
# Tasks for handling CLI customization and publishing
+#
+# Note: The configuration directory is actually located on the local machine;
+# this script is run under the local host, usually 172.17.0.1 (docker local)
-- name: "ensure nginx directory for cli exists"
+- name: "Ensure nginx directory for cli exists"
file:
- path: "{{ cli.nginxdir }}"
+ path: "{{ openwhisk_cli.nginxdir.name }}"
state: directory
- become: "{{ cli.dir.become }}"
+ become: "{{ openwhisk_cli.nginxdir.become }}"
-- set_fact:
- cli_installation_mode="{{ openwhisk_cli.installation_mode }}"
+- name: "Ensure OpenWhisk build directory exists (for temp archive work)"
+ file:
+ path: "{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}"
+ state: directory
+
+#
+# Why are we unarchiving into the build directory instead of directly into
+# the Nginx config directory? Because the Nginx config directory is (by
+# default) located in the /tmp/... directory tree, which has a sticky bit
+# set. Said sticky bit creates no end of troubles for tar, so we're going
+# to just avoid it entirely, rather than muck about with who's got which tar
+# with which right tar options installed where. It makes for many more
+# items in this ansible playbook than we'd hoped, but at least it's (fairly)
+# straightforward.
+#
+- name: "Download release archive to build directory ..."
+ get_url:
+ url: "{{ openwhisk_cli.remote.location }}/{{ openwhisk_cli.archive_name}}-{{ openwhisk_cli_tag }}-all.tgz"
+ dest: "{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}.tgz"
+ headers: "{{ openwhisk_cli.remote.headers | default('') }}"
+ when: openwhisk_cli.installation_mode == "remote"
+
+- name: "... or Copy release archive to build directory"
+ copy:
+ src: "{{ openwhisk_cli_home }}/release/{{ openwhisk_cli.archive_name}}-{{ openwhisk_cli_tag }}-all.tgz"
+ dest: "{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}.tgz"
+ when: openwhisk_cli.installation_mode == "local"
-- include: docker_login.yml
+#
+# I really wanted to use 'unarchive' here, but it was quite buggy and didn't
+# want to cooperate, so we do a good old-fashioned tar x instead
+#
+- name: "Expand the archive into the build directory"
+ shell: >
+ tar zxf /{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}.tgz
+ -C {{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}/
-- name: "download the binaries from the release page of OpenWhisk CLI"
- include: cli_remote_mode.yml
- when: cli_installation_mode == "remote"
+- name: "Copy expanded archive to final configuration directory"
+ copy:
+ # WARNING: The trailing slash is significant, signalling to copy contents
+ src: "{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}/"
+ dest: "{{ openwhisk_cli.nginxdir.name }}"
-- name: "copy the binaries from a local directory to Nginx directory"
- include: copy_local_openwhisk_cli.yml
- when: cli_installation_mode == "local"
+- name: "Delete archive from build directory"
+ file:
+ path: "{{ item }}"
+ state: absent
+ force: yes
with_items:
- - "{{ cli_os_arch }}"
+ - "{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}.tgz"
+ - "{{ openwhisk_build_dir }}/{{ openwhisk_cli.archive_name }}/"
+
+- name: "Generate a list of individual tarballs to expand"
+ find:
+ paths: "{{ openwhisk_cli.nginxdir.name }}"
+ patterns: '*.tgz'
+ recurse: true
+ register: individual_tarballs
+
+- name: "Unarchive the individual tarballs"
+ shell: tar zxf {{ item.path }} -C {{ item.path | dirname }}
+ with_items: "{{ individual_tarballs.files }}"
-- name: "copy the local content.json from a local directory to Nginx directory"
- copy: src={{ openwhisk_cli.local.location }}/content.json dest={{ cli.nginxdir }}/
- when: cli_installation_mode == "local"
+- name: "Generate a list of individual zipfiles to expand"
+ find:
+ paths: "{{ openwhisk_cli.nginxdir.name }}"
+ patterns: '*.zip'
+ recurse: true
+ register: individual_zipfiles
-- include: download_cli.yml
- when: cli_installation_mode == "remote"
+- name: "Unarchive the individual zipfiles into binaries"
+ unarchive:
+ remote_src: yes
+ src: "{{ item.path }}"
+ dest: "{{ item.path | dirname }}"
+ with_items: "{{ individual_zipfiles.files }}"
diff --git a/ansible/roles/cli/tasks/download_cli.yml b/ansible/roles/cli/tasks/download_cli.yml
deleted file mode 100644
index 324657e02f..0000000000
--- a/ansible/roles/cli/tasks/download_cli.yml
+++ /dev/null
@@ -1,32 +0,0 @@
----
-
-# Download the cli to openwhisk_home
-
-- name: "Use ansible fact to determine wsk architecture"
- set_fact:
- cli_arch: "{{ ansible_machine }}"
-
-- name: "If architecture is x86_64, change it to amd64"
- set_fact:
- cli_arch: "amd64"
- when: ansible_machine == "x86_64"
-
-- name: "download cli (linux) to openwhisk home at {{ openwhisk_home }}"
- local_action: >
- get_url
- url="https://{{ groups['edge'] | first }}/cli/go/download/linux/{{ cli_arch }}/wsk"
- dest="{{ openwhisk_home }}/bin/wsk"
- mode=0755
- validate_certs=False
- force=True
- when: ('environments/docker-machine' not in hosts_dir) and (ansible_os_family != "Darwin")
-
-- name: "download cli for mac (docker-machine or Docker for Mac) to openwhisk home at {{ openwhisk_home }}"
- local_action: >
- get_url
- url="https://{{ groups['edge'] | first }}/cli/go/download/mac/{{ cli_arch }}/wsk"
- dest="{{ openwhisk_home }}/bin/wsk"
- mode=0755
- validate_certs=False
- force=True
- when: ('environments/docker-machine' in hosts_dir ) or (ansible_os_family == "Darwin")
diff --git a/ansible/roles/cli/tasks/download_openwhisk_cli.yml b/ansible/roles/cli/tasks/download_openwhisk_cli.yml
deleted file mode 100644
index 1dff7d1836..0000000000
--- a/ansible/roles/cli/tasks/download_openwhisk_cli.yml
+++ /dev/null
@@ -1,76 +0,0 @@
----
-
-# Download the cli to Nginx CLI directory and unzip them
-
-- name: "set the basic variables os, arc and suffix"
- set_fact:
- os="{{ item[0] }}"
- arc="{{ item[1] }}"
- suffix="zip"
-
-- name: "change the suffix to tgz if the operating system is Linux"
- set_fact:
- suffix="tgz"
- when: os == "linux"
-
-- name: "set the file name to download"
- set_fact:
- source_name="{{ openwhisk_cli.remote.name }}-{{ openwhisk_cli_tag }}-{{ os }}-{{ arc }}.{{ suffix }}"
-
-- name: "set the destination file name locally (default)"
- set_fact:
- dest_name="{{ openwhisk_cli.remote.dest_name }}-{{ os }}-{{arc}}.{{ suffix }}"
-
-- name: "set the destination file name locally after download for 32 bit"
- set_fact:
- dest_name="{{ openwhisk_cli.remote.dest_name }}-{{ os }}-32bit.{{ suffix }}"
- when: arc == "386"
-
-- name: "set the destination file name locally after download for amd64"
- set_fact:
- dest_name="{{ openwhisk_cli.remote.dest_name }}-{{ os }}.{{ suffix }}"
- when: arc == "amd64"
-
-- name: "set the http header used to download the binary packages"
- set_fact:
- headers="{{ openwhisk_cli.remote.headers }}"
- when: openwhisk_cli.remote.headers is defined
-
-- name: "set the http header to empty"
- set_fact:
- headers=""
- when: openwhisk_cli.remote.headers is not defined
-
-- name: "set the volume_dir"
- vars:
- instance: "{{instances | selectattr('name', 'equalto', 'db') | list | first}}"
- set_fact:
- volume_dir: "{{ instance.volume.fsmount | default( '/mnt/' + group_names|first, true ) }}:/usr/local/var/lib/couchdb"
- when: (block_device is defined) and (block_device in disk_status.stdout)
-
-- name: "ensure Nginx cli directory for ({{ os }}) {{ arc }} exists"
- file:
- path: "{{ cli.nginxdir }}/{{ os }}/{{ arc }}"
- state: directory
- mode: 0777
- become: "{{ cli.dir.become }}"
-
-- name: "download cli ({{ os }}) {{ arc }} to Nginx directory"
- get_url:
- url: "{{ openwhisk_cli.remote.location }}/{{ source_name }}"
- dest: "{{ cli.nginxdir }}/{{ os }}/{{ arc }}/{{ dest_name }}"
- headers: "{{ headers }}"
- mode: 0755
-
-- name: "unarchive cli for Mac and Windows"
- unarchive:
- src: "{{ cli.nginxdir }}/{{ os }}/{{ arc }}/{{ dest_name }}"
- dest: "{{ cli.nginxdir }}/{{ os }}/{{ arc }}/"
- remote_src: True
- when: os != "linux"
-
-- name: "unarchive cli for Linux"
- command: tar -zxvf {{ dest_name }}
- args:
- chdir: "{{ cli.nginxdir }}/{{ os }}/{{ arc }}/"
- when: os == "linux"
diff --git a/ansible/routemgmt.yml b/ansible/routemgmt.yml
index 0c9ced2578..1807328b13 100644
--- a/ansible/routemgmt.yml
+++ b/ansible/routemgmt.yml
@@ -5,4 +5,5 @@
- hosts: ansible
roles:
+ - cli-install
- routemgmt
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services