You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@openwhisk.apache.org by GitBox <gi...@apache.org> on 2018/01/17 15:42:01 UTC
[GitHub] rabbah closed pull request #145: Use YAML files deploy kafka package to OpenWhisk on K8s
rabbah closed pull request #145: Use YAML files deploy kafka package to OpenWhisk on K8s
URL: https://github.com/apache/incubator-openwhisk-deploy-kube/pull/145
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/docker/kafkapkg-installer/Dockerfile b/docker/kafkapkg-installer/Dockerfile
new file mode 100644
index 0000000..b1960db
--- /dev/null
+++ b/docker/kafkapkg-installer/Dockerfile
@@ -0,0 +1,11 @@
+from ubuntu:latest
+
+RUN apt-get -y update && apt-get -y install \
+ git \
+ wget \
+ zip
+
+COPY init.sh /init.sh
+RUN chmod +x /init.sh
+
+CMD ["/init.sh"]
diff --git a/docker/kafkapkg-installer/init.sh b/docker/kafkapkg-installer/init.sh
new file mode 100644
index 0000000..2c30a7c
--- /dev/null
+++ b/docker/kafkapkg-installer/init.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+set -ex
+
+git clone https://github.com/apache/incubator-openwhisk-package-kafka.git
+
+# TODO: installxxxCatalog.sh wants OPENWHISK_HOME set, but doesn't actually need
+# it for anything. Fix upstream and then remove this.
+export OPENWHISK_HOME=/openwhisk
+export DB_URL=http://$DB_USERNAME:$DB_PASSWORD@$DB_HOST:$DB_PORT
+mkdir -p $OPENWHISK_HOME/bin
+
+# Download and install openwhisk cli
+pushd $OPENWHISK_HOME/bin
+ wget -q https://github.com/apache/incubator-openwhisk-cli/releases/download/latest/OpenWhisk_CLI-latest-linux-amd64.tgz
+ tar xzf OpenWhisk_CLI-latest-linux-amd64.tgz
+popd
+
+pushd /incubator-openwhisk-package-kafka
+ ./installKafka.sh $AUTH $APIHOST $DB_URL $DB_PREFIX $APIHOST
+ ./installCatalog.sh $AUTH $APIHOST $DB_URL $DB_PREFIX $APIHOST
+popd
+
+echo "successfully setup kafka package"
+
diff --git a/kubernetes/package-kafka/README.md b/kubernetes/package-kafka/README.md
new file mode 100644
index 0000000..792e551
--- /dev/null
+++ b/kubernetes/package-kafka/README.md
@@ -0,0 +1,43 @@
+# Deploy kafka package to Apache OpenWhisk
+
+This project is to deploy kafka package to local Apache OpenWhisk on a K8s using YAML file.
+
+## Prerequisite
+Edit package-kafka.env as needed to set the appropriate values for your deployment, then create the configmap packages.kafkaprovider:
+```
+kubectl -n openwhisk create cm packages.kafkaprovider --from-literal=kafkapkg_db_prefix=mq
+```
+
+The deployment also has dependencies to secret `whisk.auth` and `db.auth`, and configmap `whisk.ingress`. Make sure you have these settings before you start the deployment.
+
+## Step 1. Install kafka provider
+```
+kubectl apply -f kafkaprovider.yml
+```
+
+## Step 2. Install messaging package to your local Apache OpenWhisk
+```
+kubectl apply -f kafkapkginstaller.yml
+```
+
+## Verify your Kafka package
+Get the description of your Kafka package by:
+```
+wsk package get /whisk.system/messaging --summary -i
+```
+Create a kafka package binding:
+```
+wsk package bind /whisk.system/messaging myKafkaPkg -p brokers "[\"kafka_host1:9093\", \"kafka_host2:9093\"]" -i
+```
+Create a trigger:
+```
+wsk trigger create MyKafkaTrigger -f myKafkaPkg/kafkaFeed -p topic in-topic -i
+```
+Send a message to kafka topic by invoking the action `kafkaProduce`:
+```
+wsk action invoke myKafkaPkg/kafkaProduce -p topic in-topic -p value "this is a message" -i
+```
+Check activation log to see `MyKafkaTrigger` is triggered when a new message is sent.
+```
+wsk activation poll -i
+```
diff --git a/kubernetes/package-kafka/kafkapkginstaller.yml b/kubernetes/package-kafka/kafkapkginstaller.yml
new file mode 100644
index 0000000..6b10dc4
--- /dev/null
+++ b/kubernetes/package-kafka/kafkapkginstaller.yml
@@ -0,0 +1,48 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: kafkapkginstaller
+ namespace: openwhisk
+ labels:
+ name: kafkapkginstaller
+spec:
+ template:
+ metadata:
+ labels:
+ name: kafkapkginstaller
+ spec:
+ restartPolicy: Never
+ containers:
+ - name: kafkapkginstaller
+ imagePullPolicy: IfNotPresent
+ image: openwhisk/kube-kafkapkginstaller
+ env:
+ - name: "APIHOST"
+ valueFrom:
+ configMapKeyRef:
+ name: whisk.ingress
+ key: api_host
+ - name: "DB_HOST"
+ value: "$(COUCHDB_SERVICE_HOST)"
+ - name: "DB_PORT"
+ value: "$(COUCHDB_SERVICE_PORT_COUCHDB)"
+ - name: "DB_USERNAME"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_username
+ - name: "DB_PASSWORD"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_password
+ - name: "DB_PREFIX"
+ valueFrom:
+ configMapKeyRef:
+ name: packages.kafkaprovider
+ key: kafkapkg_db_prefix
+ - name: "AUTH"
+ valueFrom:
+ secretKeyRef:
+ name: whisk.auth
+ key: system
diff --git a/kubernetes/package-kafka/kafkaprovider.yml b/kubernetes/package-kafka/kafkaprovider.yml
new file mode 100644
index 0000000..db9c51a
--- /dev/null
+++ b/kubernetes/package-kafka/kafkaprovider.yml
@@ -0,0 +1,54 @@
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+ name: kafkaprovider
+ namespace: openwhisk
+ labels:
+ name: kafkaprovider
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ name: kafkaprovider
+ access: db
+ spec:
+ restartPolicy: Always
+ containers:
+ - name: kafkaprovider
+ imagePullPolicy: IfNotPresent
+ image: openwhisk/kafkaprovider
+ ports:
+ - name: kafkaprovider
+ containerPort: 8080
+ env:
+ - name: "DB_URL"
+ value: "http://$(COUCHDB_SERVICE_HOST):$(COUCHDB_SERVICE_PORT_COUCHDB)"
+ - name: "DB_USER"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_username
+ - name: "DB_PASS"
+ valueFrom:
+ secretKeyRef:
+ name: db.auth
+ key: db_password
+ - name: "DB_PREFIX"
+ valueFrom:
+ configMapKeyRef:
+ name: packages.kafkaprovider
+ key: kafkapkg_db_prefix
+ - name: "LOCAL_DEV"
+ value: "true"
+ - name: "ROUTER_HOST"
+ valueFrom:
+ configMapKeyRef:
+ name: whisk.ingress
+ key: api_host
+ - name: "ENDPOINT_AUTH"
+ valueFrom:
+ secretKeyRef:
+ name: whisk.auth
+ key: system
diff --git a/tools/travis/build.sh b/tools/travis/build.sh
index 35cb378..5b2dd7b 100755
--- a/tools/travis/build.sh
+++ b/tools/travis/build.sh
@@ -250,6 +250,15 @@ pushd kubernetes/openwhisk-catalog
jobHealthCheck "install-catalog"
popd
+# install package-kafka
+echo "Installing kafka package"
+pushd kubernetes/package-kafka
+ kubectl -n openwhisk create cm packages.kafkaprovider --from-literal=kafkapkg_db_prefix=mq
+ kubectl apply -f kafkaprovider.yml
+ kubectl apply -f kafkapkginstaller.yml
+ jobHealthCheck "kafkapkginstaller"
+popd
+
# list packages and actions now installed in /whisk.system
wsk -i --auth `cat kubernetes/cluster-setup/auth.whisk.system` package list
wsk -i --auth `cat kubernetes/cluster-setup/auth.whisk.system` action list
diff --git a/tools/travis/collect-logs.sh b/tools/travis/collect-logs.sh
index 0798c5f..020626f 100755
--- a/tools/travis/collect-logs.sh
+++ b/tools/travis/collect-logs.sh
@@ -20,8 +20,10 @@ kubectl -n openwhisk logs controller-1 >& logs/controller-1.log
kubectl -n openwhisk logs -lname=invoker -c docker-pull-runtimes >& logs/invoker-docker-pull.log
kubectl -n openwhisk logs -lname=invoker -c invoker >& logs/invoker-invoker.log
kubectl -n openwhisk logs -lname=nginx >& logs/nginx.log
+kubectl -n openwhisk logs -lname=kafkaprovider >& logs/kafkaprovider.log
kubectl -n openwhisk logs jobs/install-routemgmt >& logs/routemgmt.log
kubectl -n openwhisk logs jobs/install-catalog >& logs/catalog.log
+kubectl -n openwhisk logs jobs/kafkapkginstaller >& logs/kafkapkginstaller.log
kubectl get pods --all-namespaces -o wide --show-all >& logs/all-pods.txt
# System level logs from minikube
diff --git a/tools/travis/deploy.sh b/tools/travis/deploy.sh
index 11d75a0..7b2606a 100755
--- a/tools/travis/deploy.sh
+++ b/tools/travis/deploy.sh
@@ -18,3 +18,6 @@ echo "Publishing kube-openwhisk-catalog image"
echo "Publishing kube-routemgmt image"
./tools/travis/publish.sh openwhisk kube-routemgmt latest docker/routemgmt
+
+echo "Publishing kube-kafkapkginstaller image"
+./tools/travis/publish.sh openwhisk kube-kafkapkginstaller latest docker/kafapkg-installer
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services