You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/02/27 13:56:58 UTC
[GitHub] csantanapr closed pull request #72: Updated version of PR-15
csantanapr closed pull request #72: Updated version of PR-15
URL: https://github.com/apache/incubator-openwhisk-devtools/pull/72
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index 400c127..139620a 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,3 +2,5 @@ openwhisk-master
openwhisk-master*
*.iml
+
+.idea/
diff --git a/docker-compose/Makefile b/docker-compose/Makefile
index 69d73dd..86e40fe 100644
--- a/docker-compose/Makefile
+++ b/docker-compose/Makefile
@@ -17,13 +17,15 @@ DOCKER_IMAGE_PREFIX ?= openwhisk
PROJECT_HOME ?= ./openwhisk-master
WSK_CLI ?= $(PROJECT_HOME)/bin/wsk
+DOCKER_NETWORK ?= "openwhisk_default"
+
OPEN_WHISK_DB_PREFIX ?= local_
DOCKER_KERNEL ?= $(shell docker version --format "{{.Server.KernelVersion}}")
ifeq ("$(UNAME_STR)","Linux")
RUNC_BINARY ?= $(shell (which runc || which docker-runc))
DOCKER_BINARY ?= $(shell (which docker))
-else
+else
RUNC_BINARY ?= $(shell if [[ $(DOCKER_KERNEL) == *-moby || $(DOCKER_KERNEL) == *-boot2docker ]]; then (docker run --rm --privileged --pid=host debian nsenter -t 1 -m -u -n -i sh -c "which runc || which docker-runc"); else (which runc || which docker-runc); fi)
DOCKER_BINARY ?= $(shell if [[ $(DOCKER_KERNEL) == *-moby || $(DOCKER_KERNEL) == *-boot2docker ]]; then (docker run --rm --privileged --pid=host debian nsenter -t 1 -m -u -n -i sh -c "which docker"); else (which docker); fi)
endif
@@ -36,7 +38,7 @@ endif
# 2. then it starts all components using docker-compose
# 3. it runs a sample hello-world function
# To stop and cleanup the environment use: make destroy
-quick-start: download download-cli run quick-start-pause hello-world quick-start-info
+quick-start: download download-cli run hello-world quick-start-info
.PHONY: download
download:
@@ -63,7 +65,7 @@ quick-start-info:
docker:
echo "building the docker images ... "
cd $(PROJECT_HOME) && \
- ./gradlew distdocker -x :core:swift3Action:distDocker -x :core:swiftAction:distDocker
+ ./gradlew distDocker
download-cli:
echo "downloading the CLI tool ... "
@@ -81,15 +83,35 @@ download-cli:
.PHONY: run
-run: print-host check-required-ports setup start-docker-compose init-couchdb init-whisk-cli
+run: print-host check-required-ports setup start-docker-compose init-couchdb quick-start-pause init-whisk-cli init-route-mgmt init-whisk-properties
print-host:
echo "host ip address: ${DOCKER_HOST_IP}"
+.PHONY: init-whisk-properties
+init-whisk-properties:
+ # Set up whisk.properties file
+ docker run --rm -v ~/tmp/openwhisk/src:/openwhisk -w /openwhisk/ansible \
+ --network="host" -t \
+ ddragosd/ansible:2.3.1.0-debian8 \
+ ansible-playbook -i environments/docker properties.yml \
+ -e openwhisk_home=/openwhisk
+
+.PHONY: init-catalog
+init-catalog:
+ # Install Openwhisk catalog
+ curl -O ./openwhisk-catalog-master.tar.gz -L https://api.github.com/repos/openwhisk/openwhisk-catalog/tarball/master > ~/tmp/openwhisk-catalog-master.tar.gz; \
+ mkdir ~/tmp/openwhisk-catalog; \
+ tar -xvf ~/tmp/openwhisk-catalog-master.tar.gz --strip 1 -C ~/tmp/openwhisk-catalog; \
+ OPENWHISK_HOME=~/tmp/openwhisk/src ~/tmp/openwhisk-catalog/packages/installCatalog.sh \
+ `cat $(PROJECT_HOME)/ansible/files/auth.whisk.system` \
+ $(DOCKER_HOST_IP):443 \
+ $(WSK_CLI)
+
.PHONY: check-required-ports
check-required-ports:
echo "checking required ports ... "
- for port in 80 443 2888 5984 8085 8888 9092 2888 8001; do \
+ for port in 80 443 2888 5984 8085 8888 9092 2888 8001 9000 9001; do \
pid=`lsof -Pi :$$port -sTCP:LISTEN -t` ; \
if [ ! -z "$$pid" ]; then echo "$$(tput setaf 1)Port $$port is taken by PID:$$pid.$$(tput sgr0)"; exit 1; fi; \
done
@@ -97,14 +119,22 @@ check-required-ports:
.PHONY: setup
setup:
+ # Put apigateway files in place
mkdir -p ~/tmp/openwhisk/apigateway/ssl
cd $(PROJECT_HOME)/ansible/roles/nginx/files/ && ./genssl.sh $(DOCKER_HOST_IP) server
cp $(PROJECT_HOME)/ansible/roles/nginx/files/*.pem ~/tmp/openwhisk/apigateway/ssl
cp -r ./apigateway/* ~/tmp/openwhisk/apigateway/
+
+ # Put project files in place
+ rsync -a $(PROJECT_HOME)/* ~/tmp/openwhisk/src --exclude .git --exclude build --exclude tests
+ cp -r ./ansible/* ~/tmp/openwhisk/src/ansible
+
+ # Write local environment file
> ~/tmp/openwhisk/local.env
printf "DOCKER_BINARY=$(DOCKER_BINARY)\n" >> ~/tmp/openwhisk/local.env
printf "RUNC_BINARY=$(RUNC_BINARY)\n" >> ~/tmp/openwhisk/local.env
printf "DOCKER_COMPOSE_HOST=$(DOCKER_HOST_IP)\n" >> ~/tmp/openwhisk/local.env
+ printf "DOCKER_NETWORK=$(DOCKER_NETWORK)\n" >> ~/tmp/openwhisk/local.env
printf "DOCKER_REGISTRY=$(DOCKER_REGISTRY)\n" >> ~/tmp/openwhisk/local.env
printf "DOCKER_IMAGE_PREFIX=$(DOCKER_IMAGE_PREFIX)\n" >> ~/tmp/openwhisk/local.env
@@ -142,23 +172,35 @@ rm:
$(shell cat ~/tmp/openwhisk/local.env) docker-compose --project-name openwhisk rm
.PHONY: init-couchdb
-init-couchdb:
+init-couchdb: setup
echo "waiting for the database to come up ... on $(DOCKER_HOST_IP)"
until $$(curl --output /dev/null --silent --head --fail http://$(DOCKER_HOST_IP):5984/_all_dbs); do printf '.'; sleep 5; done
+
echo "initializing the database ... on $(DOCKER_HOST_IP)"
- # make sure the src files are in a shared folder for docker
- mkdir -p ~/tmp/openwhisk
- rm -rf ~/tmp/openwhisk/src
- rsync -a $(PROJECT_HOME)/* ~/tmp/openwhisk/src --exclude .git --exclude build --exclude tests
- echo 'Setting up db using ansible container....'; \
- compose_network=$$(docker-compose --project-name openwhisk ps -q db 2>/dev/null | xargs docker inspect -f '{{range $$index, $$element := .NetworkSettings.Networks}}{{$$index}}{{end}}' | head -n 1); \
- db_ip_address=$$(docker-compose --project-name openwhisk ps -q db 2>/dev/null | xargs docker inspect -f "{{.NetworkSettings.Networks.$${compose_network}.IPAddress}}"); \
+
+ # Run database initialisation
+ db_ip_address=$$(docker-compose --project-name openwhisk ps -q db 2>/dev/null | xargs docker inspect -f "{{.NetworkSettings.Networks.$(DOCKER_NETWORK).IPAddress}}"); \
docker run --rm -v ~/tmp/openwhisk/src:/openwhisk -w /openwhisk/ansible \
- --network="$${compose_network}" -t \
- --add-host="db:$${db_ip_address}" \
- ddragosd/ansible:2.3.1.0-debian8 \
- sh -c "ansible-playbook setup.yml && ansible-playbook couchdb.yml --tags=ini && ansible-playbook initdb.yml wipe.yml \
- -e db_host=db -e openwhisk_home=/openwhisk -e db_prefix=$(OPEN_WHISK_DB_PREFIX)"
+ --network=$(DOCKER_NETWORK) -t \
+ --add-host="db.docker:$${db_ip_address}" \
+ ddragosd/ansible:2.3.1.0-debian8 \
+ sh -c "ansible-playbook -i environments/docker setup.yml \
+ && ansible-playbook -i environments/docker couchdb.yml \
+ && ansible-playbook -i environments/docker initdb.yml wipe.yml \
+ -e openwhisk_home=/openwhisk -e db_host=db.docker -e db_prefix=$(OPEN_WHISK_DB_PREFIX)"
+
+.PHONY: init-route-mgmt
+init-route-mgmt:
+ $(PROJECT_HOME)/ansible/roles/routemgmt/files/uninstallRouteMgmt.sh \
+ `cat $(PROJECT_HOME)/ansible/files/auth.whisk.system` \
+ $(DOCKER_HOST_IP):443 \
+ '/whisk.system' \
+ $(WSK_CLI)
+ OPENWHISK_HOME=~/tmp/openwhisk/src $(PROJECT_HOME)/ansible/roles/routemgmt/files/installRouteMgmt.sh \
+ `cat $(PROJECT_HOME)/ansible/files/auth.whisk.system` \
+ $(DOCKER_HOST_IP):443 \
+ '/whisk.system' \
+ $(WSK_CLI)
.PHONY: init-whisk-cli
init-whisk-cli:
@@ -167,6 +209,9 @@ init-whisk-cli:
echo "initializing CLI ... "
$(WSK_CLI) -v property set --namespace guest --auth `cat $(PROJECT_HOME)/ansible/files/auth.guest` --apihost $(DOCKER_HOST_IP):443 -i
+ # Add dummy API gateway access token to avoid auth errors
+ printf "APIGW_ACCESS_TOKEN=foobar\n" >> ~/.wskprops
+
.PHONY: destroy
destroy: stop rm
echo "cleaning other openwhisk containers started by the invoker ... "
diff --git a/docker-compose/README.md b/docker-compose/README.md
index 02e24f9..478c833 100644
--- a/docker-compose/README.md
+++ b/docker-compose/README.md
@@ -149,5 +149,14 @@ These 2 variable allow you to execute a JS action using the container `registry.
## Local Docker containers for controllers and invokers
By default this setup uses published images for controller and invokers from `openwhisk` namespace i.e.
-`openwhisk/controller` and `openwhisk/invoker`. To make use of locally build images you can use `DOCKER_OW_IMAGE_PREFIX`
-variable i.e. `DOCKER_OW_IMAGE_PREFIX=whisk make quick-start`
\ No newline at end of file
+`openwhisk/controller` and `openwhisk/invoker`. To make use of locally build images you can use `DOCKER_IMAGE_PREFIX`
+variable i.e. `DOCKER_IMAGE_PREFIX=whisk make quick-start`
+
+## Openwhisk catalog
+
+If you would like to install the [Openwhisk Catalog](https://github.com/apache/incubator-openwhisk-catalog),
+you need to run:
+
+```
+make init-catalog
+```
diff --git a/docker-compose/ansible/couchdb.yml b/docker-compose/ansible/couchdb.yml
new file mode 100644
index 0000000..4fad83c
--- /dev/null
+++ b/docker-compose/ansible/couchdb.yml
@@ -0,0 +1,9 @@
+---
+# This overrides the standard Openwhisk CouchDB playbook for the docker-compose environment.
+# The standard playbook includes a play running against the DB host which requires gathering facts
+# (and hence SSH access) which we don't have in the docker-compose world.
+
+- hosts: localhost
+ tasks:
+ - name: prepare db_local.ini
+ local_action: template src="db_local.ini.j2" dest="{{ playbook_dir }}/db_local.ini"
diff --git a/docker-compose/ansible/environments/docker/group_vars/all b/docker-compose/ansible/environments/docker/group_vars/all
new file mode 100644
index 0000000..b2fc2eb
--- /dev/null
+++ b/docker-compose/ansible/environments/docker/group_vars/all
@@ -0,0 +1,53 @@
+---
+
+whisk_version_name: local
+config_root_dir: /openwhisk
+whisk_logs_dir: /logs
+nginx_conf_dir: /etc/api-gateway/generated-conf.d
+docker_registry: ""
+docker_dns: ""
+bypass_pull_for_local_images: true
+
+db_prefix: "local_"
+
+db_provider: CouchDB
+db_port: 5984
+db_protocol: http
+db_username: whisk_admin
+db_password: some_passw0rd
+db_host: "{{ groups['db'] | first }}"
+
+# API GW connection configuration
+apigw_auth_user: ""
+apigw_auth_pwd: ""
+apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2"
+
+limits:
+ actions:
+ invokes:
+ perMinute: 60
+ concurrent: 30
+ concurrentInSystem: 5000
+ triggers:
+ fires:
+ perMinute: 60
+
+# Switch on runc
+invoker_use_runc: true
+
+catalog_repos:
+ openwhisk-catalog:
+ url: https://github.com/openwhisk/openwhisk-catalog.git
+ # Set the local location as the same level as openwhisk home, but it can be changed.
+ location: "{{ openwhisk_home }}/../openwhisk-catalog"
+ version: "HEAD"
+ repo_update: "no"
+
+# Set kafka configuration
+kafka_heap: '512m'
+kafka_topics_completed_retentionBytes: 104857600
+kafka_topics_completed_retentionMS: 300000
+kafka_topics_health_retentionBytes: 104857600
+kafka_topics_health_retentionMS: 300000
+kafka_topics_invoker_retentionBytes: 104857600
+kafka_topics_invoker_retentionMS: 300000
diff --git a/docker-compose/ansible/environments/docker/hosts b/docker-compose/ansible/environments/docker/hosts
new file mode 100644
index 0000000..88fd11b
--- /dev/null
+++ b/docker-compose/ansible/environments/docker/hosts
@@ -0,0 +1,25 @@
+ansible ansible_connection=local
+
+[edge]
+apigateway.docker ansible_host=apigateway.docker
+
+[controllers]
+controllers.docker ansible_host=controllers.docker
+
+[kafkas]
+kafka.docker ansible_host=kafka.docker
+
+[zookeepers]
+zookeeper.docker ansible_host=zookeeper.docker
+
+[invokers]
+invoker.docker ansible_host=invoker.docker
+
+[db]
+db.docker ansible_host=db.docker
+
+[redis]
+redis.docker ansible_host=redis.docker
+
+[apigateway]
+apigateway.docker ansible_host=apigateway.docker
diff --git a/docker-compose/apigateway/conf/whisk-docker-compose.conf b/docker-compose/apigateway/conf/whisk-docker-compose.conf
index 51857d6..8a85c6a 100644
--- a/docker-compose/apigateway/conf/whisk-docker-compose.conf
+++ b/docker-compose/apigateway/conf/whisk-docker-compose.conf
@@ -3,7 +3,7 @@
# NOTE: This upstream should come from a service discovery component
# All the upstreams could be generated sepparately and only included in the API Gateway
upstream whisk_controller {
- server whisk.controller:8888;
+ server controller.docker:8888;
keepalive 16;
}
diff --git a/docker-compose/docker-compose.yml b/docker-compose/docker-compose.yml
index 94c813a..6dc556c 100644
--- a/docker-compose/docker-compose.yml
+++ b/docker-compose/docker-compose.yml
@@ -37,7 +37,7 @@ services:
# WHISK CONTROLLER
controller:
- image: ${DOCKER_OW_IMAGE_PREFIX:-openwhisk}/controller
+ image: ${DOCKER_IMAGE_PREFIX:-openwhisk}/controller
command: /bin/sh -c "controller/bin/controller 0 >> /logs/controller-local_logs.log 2>&1"
links:
- db:db.docker
@@ -76,7 +76,7 @@ services:
# WHISK INVOKER AGENT
invoker:
- image: ${DOCKER_OW_IMAGE_PREFIX:-openwhisk}/invoker
+ image: ${DOCKER_IMAGE_PREFIX:-openwhisk}/invoker
command: /bin/sh -c "/invoker/bin/invoker 0 >> /logs/invoker-local_logs.log 2>&1"
privileged: true
pid: "host"
@@ -109,7 +109,7 @@ services:
EDGE_HOST: ${DOCKER_COMPOSE_HOST}
EDGE_HOST_APIPORT: 443
- INVOKER_CONTAINER_NETWORK: openwhisk_default
+ INVOKER_CONTAINER_NETWORK: ${DOCKER_NETWORK}
WHISK_API_HOST_NAME: ${DOCKER_COMPOSE_HOST}
volumes:
@@ -129,20 +129,32 @@ services:
expose:
- "6379"
apigateway:
- image: adobeapiplatform/apigateway:1.1.0
+ image: openwhisk/apigateway:latest
links:
- - controller:whisk.controller
+ - controller:controller.docker
- redis:redis.docker
depends_on:
- controller
- redis
+ environment:
+ PUBLIC_MANAGEDURL_HOST: ${DOCKER_COMPOSE_HOST}
+ PUBLIC_MANAGEDURL_PORT: 9001
+
+ REDIS_HOST: redis.docker
+ REDIS_PORT: 6379
volumes:
- ~/tmp/openwhisk/apigateway/ssl:/etc/ssl:ro
- ~/tmp/openwhisk/apigateway/conf:/etc/api-gateway/generated-conf.d:ro
- ~/tmp/openwhisk/apigateway/env.conf:/etc/api-gateway/environment.conf.d:ro
+ networks:
+ default:
+ aliases:
+ - apigateway.docker
ports:
- "80:80"
- "443:443"
+ - "9000:9000"
+ - "9001:8080"
kafka-rest:
image: confluentinc/cp-kafka-rest:3.3.1
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services