You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2016/08/30 07:18:53 UTC
airavata git commit: Add more docker files for kafka, logstash
Repository: airavata
Updated Branches:
refs/heads/lahiru/airavata-docker 91afea32d -> 54fbd4f78
Add more docker files for kafka, logstash
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/54fbd4f7
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/54fbd4f7
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/54fbd4f7
Branch: refs/heads/lahiru/airavata-docker
Commit: 54fbd4f789b9ea79b17046643133f8c169899a5c
Parents: 91afea3
Author: Lahiru Ginnaliya Gamathige <la...@apache.org>
Authored: Tue Aug 30 00:18:35 2016 -0700
Committer: Lahiru Ginnaliya Gamathige <la...@apache.org>
Committed: Tue Aug 30 00:18:35 2016 -0700
----------------------------------------------------------------------
deploy/images/kafka/Dockerfile | 8 ++
deploy/images/kafka/start-kafka.sh | 17 ++++
deploy/images/logstash/Dockerfile | 6 ++
.../logstash-output-amazon_es-0.3-java.gem | Bin 0 -> 22016 bytes
.../logstash-output-dogstatsd-2.0.0.gem | Bin 0 -> 9728 bytes
deploy/images/logstash/logstash.conf | 86 +++++++++++++++++++
6 files changed, 117 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/54fbd4f7/deploy/images/kafka/Dockerfile
----------------------------------------------------------------------
diff --git a/deploy/images/kafka/Dockerfile b/deploy/images/kafka/Dockerfile
new file mode 100644
index 0000000..59770e1
--- /dev/null
+++ b/deploy/images/kafka/Dockerfile
@@ -0,0 +1,8 @@
+FROM scigap/java:8
+ENV ZOOKEEPER localhost:2181/kafka
+ENV LOG_DIRS /var/lib/kafka
+ENV JMX_PORT 9999
+ADD start-kafka.sh /start-kafka.sh
+RUN mkdir -p /opt/kafka && curl http://mirrors.sonic.net/apache/kafka/0.8.2.2/kafka_2.11-0.8.2.2.tgz > /tmp/kafka.tgz && \
+ tar -zxf /tmp/kafka.tgz -C /opt/kafka --strip-components=1 && rm -f /tmp/kafka.tgz
+ENTRYPOINT /start-kafka.sh
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/54fbd4f7/deploy/images/kafka/start-kafka.sh
----------------------------------------------------------------------
diff --git a/deploy/images/kafka/start-kafka.sh b/deploy/images/kafka/start-kafka.sh
new file mode 100644
index 0000000..3b3ee86
--- /dev/null
+++ b/deploy/images/kafka/start-kafka.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+if test -z "${BROKER_ID}"; then
+ BROKER_ID=$(ifconfig eth0 | awk '/inet addr/{print substr($2,6)}' | head -n1 | tr -d '.')
+fi
+
+mkdir -p /opt/kafka/etc
+cat <<EOF > /opt/kafka/etc/server.properties
+broker.id=${BROKER_ID}
+zookeeper.connect=${ZOOKEEPER}
+log.dirs=${LOG_DIRS}
+num.partitions=2
+default.replication.factor=2
+advertised.host.name=${ADVERTISED_HOST_NAME}
+EOF
+
+exec /opt/kafka/bin/kafka-server-start.sh /opt/kafka/etc/server.properties
http://git-wip-us.apache.org/repos/asf/airavata/blob/54fbd4f7/deploy/images/logstash/Dockerfile
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/Dockerfile b/deploy/images/logstash/Dockerfile
new file mode 100644
index 0000000..c3b8963
--- /dev/null
+++ b/deploy/images/logstash/Dockerfile
@@ -0,0 +1,6 @@
+FROM logstash:2
+
+COPY *.gem /tmp/
+RUN logstash-plugin install /tmp/*.gem && rm -f /tmp/*.gem
+RUN logstash-plugin install logstash-codec-avro logstash-codec-cloudtrail logstash-input-journald
+ENTRYPOINT []
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/54fbd4f7/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem b/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem
new file mode 100644
index 0000000..d3c913a
Binary files /dev/null and b/deploy/images/logstash/logstash-output-amazon_es-0.3-java.gem differ
http://git-wip-us.apache.org/repos/asf/airavata/blob/54fbd4f7/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem b/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem
new file mode 100644
index 0000000..ec177a6
Binary files /dev/null and b/deploy/images/logstash/logstash-output-dogstatsd-2.0.0.gem differ
http://git-wip-us.apache.org/repos/asf/airavata/blob/54fbd4f7/deploy/images/logstash/logstash.conf
----------------------------------------------------------------------
diff --git a/deploy/images/logstash/logstash.conf b/deploy/images/logstash/logstash.conf
new file mode 100644
index 0000000..1f94cb2
--- /dev/null
+++ b/deploy/images/logstash/logstash.conf
@@ -0,0 +1,86 @@
+input {
+ kafka {
+ topic_id = > "${ENV_NAME}_all_logs"
+ zk_connect = > "${KAFKA_ZK}"
+ auto_offset_reset = > "smallest"
+ type = > "all_logs"
+ }
+ kafka {
+ topic_id = > "${ENV_NAME}_apiserver_logs"
+ zk_connect = > "${KAFKA_ZK}"
+ auto_offset_reset = > "smallest"
+ type = > "apiserver_logs"
+ }
+ kafka {
+ topic_id = > "${ENV_NAME}_gfac_logs"
+ zk_connect = > "${KAFKA_ZK}"
+ auto_offset_reset = > "smallest"
+ type = > "gfac_logs"
+ }
+ kafka {
+ topic_id = > "${ENV_NAME}_orchestrator_logs"
+ zk_connect = > "${KAFKA_ZK}"
+ auto_offset_reset = > "smallest"
+ type = > "orchestrator_logs"
+ }
+ kafka {
+ topic_id = > "${ENV_NAME}_credentialstore_logs"
+ zk_connect = > "${KAFKA_ZK}"
+ auto_offset_reset = > "smallest"
+ type = > "credentialstore_logs"
+ }
+}
+
+filter {
+ mutate {add_field = > {"[@metadata][level]" = > "%{[level]}"}}
+ mutate {lowercase = > ["[@metadata][level]"]}
+ mutate {gsub = > ["level", "LOG_", ""]}
+ mutate {
+ add_tag = > ["${ENV_NAME}", "${OS_NAME}-${OS_VERSION}"]
+ }
+ date {
+ match = > ["timestamp", "ISO8601"]
+ }
+}
+
+output {
+ if [type] == "apiserver_logs" {
+ if [@metadata][level] == "debug" {
+ amazon_es {
+ hosts = > ["${ES_ENDPOINT}"]
+ region = > "us-east-1"
+ index = > "${ENV_NAME}-apiserver-logs-logstash-%{+YYYY.MM.dd}"
+ }
+ }
+ } else if [type] == "gfac_logs" {
+ if [@metadata][level] == "debug" {
+ amazon_es {
+ hosts = > ["${ES_ENDPOINT}"]
+ region = > "us-east-1"
+ index = > "${ENV_NAME}-gfac-logs-logstash-%{+YYYY.MM.dd}"
+ }
+ }
+ } else if [type] == "orchestrator_logs" {
+ if [@metadata][level] == "debug" {
+ amazon_es {
+ hosts = > ["${ES_ENDPOINT}"]
+ region = > "us-east-1"
+ index = > "${ENV_NAME}-orchestrator-logs-logstash-%{+YYYY.MM.dd}"
+ }
+ }
+ } else if [type] == "credentialstore_logs" {
+ if [@metadata][level] == "debug" {
+ amazon_es {
+ hosts = > ["${ES_ENDPOINT}"]
+ region = > "us-east-1"
+ index = > "${ENV_NAME}-credentialstore-logs-logstash-%{+YYYY.MM.dd}"
+ }
+ }
+ } else {
+ amazon_es {
+ hosts => ["${ES_ENDPOINT}"]
+ region => "us-east-1"
+ index => "${ENV_NAME}-airavata-logs-logstash-%{+YYYY.MM.dd}"
+ }
+ }
+}