You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@openwhisk.apache.org by ra...@apache.org on 2018/01/17 15:41:59 UTC
[incubator-openwhisk-deploy-kube] branch master updated: add YAML
files to deploy kafka package (#145)
This is an automated email from the ASF dual-hosted git repository.
rabbah pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-openwhisk-deploy-kube.git
The following commit(s) were added to refs/heads/master by this push:
new bfd2036 add YAML files to deploy kafka package (#145)
bfd2036 is described below
commit bfd2036964cea08f492162d5b661e38549829bee
Author: Ying Chun Guo <gu...@cn.ibm.com>
AuthorDate: Wed Jan 17 23:41:57 2018 +0800
add YAML files to deploy kafka package (#145)
---
docker/kafkapkg-installer/Dockerfile | 11 ++++++
docker/kafkapkg-installer/init.sh | 24 ++++++++++++
kubernetes/package-kafka/README.md | 43 ++++++++++++++++++++
kubernetes/package-kafka/kafkapkginstaller.yml | 48 +++++++++++++++++++++++
kubernetes/package-kafka/kafkaprovider.yml | 54 ++++++++++++++++++++++++++
tools/travis/build.sh | 9 +++++
tools/travis/collect-logs.sh | 2 +
tools/travis/deploy.sh | 3 ++
8 files changed, 194 insertions(+)
diff --git a/docker/kafkapkg-installer/Dockerfile b/docker/kafkapkg-installer/Dockerfile
new file mode 100644
index 0000000..b1960db
--- /dev/null
+++ b/docker/kafkapkg-installer/Dockerfile
@@ -0,0 +1,11 @@
+from ubuntu:latest
+
+RUN apt-get -y update && apt-get -y install \
+ git \
+ wget \
+ zip
+
+COPY init.sh /init.sh
+RUN chmod +x /init.sh
+
+CMD ["/init.sh"]
diff --git a/docker/kafkapkg-installer/init.sh b/docker/kafkapkg-installer/init.sh
new file mode 100644
index 0000000..2c30a7c
--- /dev/null
+++ b/docker/kafkapkg-installer/init.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -ex
+
+git clone https://github.com/apache/incubator-openwhisk-package-kafka.git
+
+# TODO: installxxxCatalog.sh wants OPENWHISK_HOME set, but doesn't actually need
+# it for anything. Fix upstream and then remove this.
+export OPENWHISK_HOME=/openwhisk
+export DB_URL=http://$DB_USERNAME:$DB_PASSWORD@$DB_HOST:$DB_PORT
+mkdir -p $OPENWHISK_HOME/bin
+
+# Download and install openwhisk cli
+pushd $OPENWHISK_HOME/bin
+ wget -q https://github.com/apache/incubator-openwhisk-cli/releases/download/latest/OpenWhisk_CLI-latest-linux-amd64.tgz
+ tar xzf OpenWhisk_CLI-latest-linux-amd64.tgz
+popd
+
+pushd /incubator-openwhisk-package-kafka
+ ./installKafka.sh $AUTH $APIHOST $DB_URL $DB_PREFIX $APIHOST
+ ./installCatalog.sh $AUTH $APIHOST $DB_URL $DB_PREFIX $APIHOST
+popd
+
+echo "successfully setup kafka package"
+
diff --git a/kubernetes/package-kafka/README.md b/kubernetes/package-kafka/README.md
new file mode 100644
index 0000000..792e551
--- /dev/null
+++ b/kubernetes/package-kafka/README.md
@@ -0,0 +1,43 @@
+# Deploy kafka package to Apache OpenWhisk
+
+This project is to deploy kafka package to local Apache OpenWhisk on a K8s using YAML file.
+
+## Prerequisite
+Edit package-kafka.env as needed to set the appropriate values for your deployment, then create the configmap packages.kafkaprovider:
+```
+kubectl -n openwhisk create cm packages.kafkaprovider --from-literal=kafkapkg_db_prefix=mq
+```
+
+The deployment also has dependencies to secret `whisk.auth` and `db.auth`, and configmap `whisk.ingress`. Make sure you have these settings before you start the deployment.
+
+## Step 1. Install kafka provider
+```
+kubectl apply -f kafkaprovider.yml
+```
+
+## Step 2. Install messaging package to your local Apache OpenWhisk
+```
+kubectl apply -f kafkapkginstaller.yml
+```
+
+## Verify your Kafka package
+Get the description of your Kafka package by:
+```
+wsk package get /whisk.system/messaging --summary -i
+```
+Create a kafka package binding:
+```
+wsk package bind /whisk.system/messaging myKafkaPkg -p brokers "[\"kafka_host1:9093\", \"kafka_host2:9093\"]" -i
+```
+Create a trigger:
+```
+wsk trigger create MyKafkaTrigger -f myKafkaPkg/kafkaFeed -p topic in-topic -i
+```
+Send a message to kafka topic by invoking the action `kafkaProduce`:
+```
+wsk action invoke myKafkaPkg/kafkaProduce -p topic in-topic -p value "this is a message" -i
+```
+Check activation log to see `MyKafkaTrigger` is triggered when a new message is sent.
+```
+wsk activation poll -i
+```
diff --git a/kubernetes/package-kafka/kafkapkginstaller.yml b/kubernetes/package-kafka/kafkapkginstaller.yml
new file mode 100644
index 0000000..6b10dc4
--- /dev/null
+++ b/kubernetes/package-kafka/kafkapkginstaller.yml
@@ -0,0 +1,48 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: kafkapkginstaller
+ namespace: openwhisk
+ labels:
+ name: kafkapkginstaller
+spec:
+ template:
+ metadata:
+ labels:
+ name: kafkapkginstaller
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: kafkapkginstaller
+ imagePullPolicy: IfNotPresent
+ image: openwhisk/kube-kafkapkginstaller
+ env:
+ - name: "APIHOST"
+ valueFrom:
+ configMapKeyRef:
+ name: whisk.ingress
+ key: api_host
+ - name: "DB_HOST"
+ value: "$(COUCHDB_SERVICE_HOST)"
+ - name: "DB_PORT"
+ value: "$(COUCHDB_SERVICE_PORT_COUCHDB)"
+ - name: "DB_USERNAME"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_username
+ - name: "DB_PASSWORD"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_password
+ - name: "DB_PREFIX"
+ valueFrom:
+ configMapKeyRef:
+ name: packages.kafkaprovider
+ key: kafkapkg_db_prefix
+ - name: "AUTH"
+ valueFrom:
+ secretKeyRef:
+ name: whisk.auth
+ key: system
diff --git a/kubernetes/package-kafka/kafkaprovider.yml b/kubernetes/package-kafka/kafkaprovider.yml
new file mode 100644
index 0000000..db9c51a
--- /dev/null
+++ b/kubernetes/package-kafka/kafkaprovider.yml
@@ -0,0 +1,54 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: kafkaprovider
+ namespace: openwhisk
+ labels:
+ name: kafkaprovider
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ name: kafkaprovider
+ access: db
+ spec:
+ restartPolicy: Always
+ containers:
+ - name: kafkaprovider
+ imagePullPolicy: IfNotPresent
+ image: openwhisk/kafkaprovider
+ ports:
+ - name: kafkaprovider
+ containerPort: 8080
+ env:
+ - name: "DB_URL"
+ value: "http://$(COUCHDB_SERVICE_HOST):$(COUCHDB_SERVICE_PORT_COUCHDB)"
+ - name: "DB_USER"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_username
+ - name: "DB_PASS"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_password
+ - name: "DB_PREFIX"
+ valueFrom:
+ configMapKeyRef:
+ name: packages.kafkaprovider
+ key: kafkapkg_db_prefix
+ - name: "LOCAL_DEV"
+ value: "true"
+ - name: "ROUTER_HOST"
+ valueFrom:
+ configMapKeyRef:
+ name: whisk.ingress
+ key: api_host
+ - name: "ENDPOINT_AUTH"
+ valueFrom:
+ secretKeyRef:
+ name: whisk.auth
+ key: system
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index a6de7b5..3920d80 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -250,6 +250,15 @@ pushd kubernetes/openwhisk-catalog
jobHealthCheck "install-catalog"
popd
+# install package-kafka
+echo "Installing kafka package"
+pushd kubernetes/package-kafka
+ kubectl -n openwhisk create cm packages.kafkaprovider --from-literal=kafkapkg_db_prefix=mq
+ kubectl apply -f kafkaprovider.yml
+ kubectl apply -f kafkapkginstaller.yml
+ jobHealthCheck "kafkapkginstaller"
+popd
+
# list packages and actions now installed in /whisk.system
wsk -i --auth `cat kubernetes/cluster-setup/auth.whisk.system` package list
wsk -i --auth `cat kubernetes/cluster-setup/auth.whisk.system` action list
diff --git a/tools/travis/collect-logs.sh b/tools/travis/collect-logs.sh
index 0798c5f..020626f 100755
--- a/tools/travis/collect-logs.sh
+++ b/tools/travis/collect-logs.sh
@@ -20,8 +20,10 @@ kubectl -n openwhisk logs controller-1 >& logs/controller-1.log
kubectl -n openwhisk logs -lname=invoker -c docker-pull-runtimes >& logs/invoker-docker-pull.log
kubectl -n openwhisk logs -lname=invoker -c invoker >& logs/invoker-invoker.log
kubectl -n openwhisk logs -lname=nginx >& logs/nginx.log
+kubectl -n openwhisk logs -lname=kafkaprovider >& logs/kafkaprovider.log
kubectl -n openwhisk logs jobs/install-routemgmt >& logs/routemgmt.log
kubectl -n openwhisk logs jobs/install-catalog >& logs/catalog.log
+kubectl -n openwhisk logs jobs/kafkapkginstaller >& logs/kafkapkginstaller.log
kubectl get pods --all-namespaces -o wide --show-all >& logs/all-pods.txt
# System level logs from minikube
diff --git a/tools/travis/deploy.sh b/tools/travis/deploy.sh
index 11d75a0..7b2606a 100755
--- a/tools/travis/deploy.sh
+++ b/tools/travis/deploy.sh
@@ -18,3 +18,6 @@ echo "Publishing kube-openwhisk-catalog image"
echo "Publishing kube-routemgmt image"
./tools/travis/publish.sh openwhisk kube-routemgmt latest docker/routemgmt
+
+echo "Publishing kube-kafkapkginstaller image"
+./tools/travis/publish.sh openwhisk kube-kafkapkginstaller latest docker/kafapkg-installer
--
To stop receiving notification emails like this one, please contact
['"commits@openwhisk.apache.org" <co...@openwhisk.apache.org>'].