You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2016/09/29 12:14:43 UTC
airavata git commit: Ddog related deployment
Repository: airavata
Updated Branches:
refs/heads/lahiru/airavata-docker b338041db -> 805fc852e
Ddog related deployment
Project: http://git-wip-us.apache.org/repos/asf/airavata/repo
Commit: http://git-wip-us.apache.org/repos/asf/airavata/commit/805fc852
Tree: http://git-wip-us.apache.org/repos/asf/airavata/tree/805fc852
Diff: http://git-wip-us.apache.org/repos/asf/airavata/diff/805fc852
Branch: refs/heads/lahiru/airavata-docker
Commit: 805fc852e62b91c2a0768c91b85bc2c32594f578
Parents: b338041
Author: Lahiru Ginnaliya Gamathige <la...@highfive.com>
Authored: Thu Sep 29 05:13:06 2016 -0700
Committer: Lahiru Ginnaliya Gamathige <la...@highfive.com>
Committed: Thu Sep 29 05:13:06 2016 -0700
----------------------------------------------------------------------
deploy/images/datadog/Dockerfile | 18 +++
deploy/images/datadog/conf.d/elastic.yaml | 6 +
deploy/images/datadog/conf.d/kafka.yaml | 158 +++++++++++++++++++
.../images/datadog/conf.d/logstash-process.yaml | 8 +
deploy/images/datadog/conf.d/zk.yaml | 6 +
5 files changed, 196 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/airavata/blob/805fc852/deploy/images/datadog/Dockerfile
----------------------------------------------------------------------
diff --git a/deploy/images/datadog/Dockerfile b/deploy/images/datadog/Dockerfile
new file mode 100644
index 0000000..42ce627
--- /dev/null
+++ b/deploy/images/datadog/Dockerfile
@@ -0,0 +1,18 @@
+FROM datadog/docker-dd-agent:latest
+
+ENV JAVA_MAJOR 8
+ENV JAVA_MINOR 72
+ENV JAVA_BUILD 15
+
+RUN apt-get update \
+ && apt-get install -y net-tools curl ca-certificates \
+ && cd /opt \
+ && curl -sSLH "Cookie: oraclelicense=accept-securebackup-cookie" \
+ "http://download.oracle.com/otn-pub/java/jdk/${JAVA_MAJOR}u${JAVA_MINOR}-b${JAVA_BUILD}/server-jre-${JAVA_MAJOR}u${JAVA_MINOR}-linux-x64.tar.gz" -o jdk.tar.gz \
+ && tar -zxf jdk.tar.gz \
+ && update-alternatives --install /usr/bin/java java /opt/jdk1.$JAVA_MAJOR.0_$JAVA_MINOR/bin/java 100 \
+ && rm jdk.tar.gz \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+COPY conf.d/ /etc/dd-agent/conf.d/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/805fc852/deploy/images/datadog/conf.d/elastic.yaml
----------------------------------------------------------------------
diff --git a/deploy/images/datadog/conf.d/elastic.yaml b/deploy/images/datadog/conf.d/elastic.yaml
new file mode 100644
index 0000000..17d65aa
--- /dev/null
+++ b/deploy/images/datadog/conf.d/elastic.yaml
@@ -0,0 +1,6 @@
+init_config:
+
+instances:
+ - url: http://localhost:9200
+ cluster_stats: false
+ pshard_stats: true
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/805fc852/deploy/images/datadog/conf.d/kafka.yaml
----------------------------------------------------------------------
diff --git a/deploy/images/datadog/conf.d/kafka.yaml b/deploy/images/datadog/conf.d/kafka.yaml
new file mode 100644
index 0000000..112e889
--- /dev/null
+++ b/deploy/images/datadog/conf.d/kafka.yaml
@@ -0,0 +1,158 @@
+instances:
+ - host: localhost
+ port: 9999
+
+init_config:
+ is_jmx: true
+
+ # Metrics collected by this check. You should not have to modify this.
+ conf:
+ #
+ # Aggregate cluster stats
+ #
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesOutPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.net.bytes_out
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesInPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.net.bytes_in
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=BrokerTopicMetrics,name=MessagesInPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.messages_in
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=BrokerTopicMetrics,name=BytesRejectedPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.net.bytes_rejected
+
+ #
+ # Request timings
+ #
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedFetchRequestsPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.request.fetch.failed
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=BrokerTopicMetrics,name=FailedProduceRequestsPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.request.produce.failed
+ - include:
+ domain: 'kafka.network'
+ bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Produce'
+ attribute:
+ Mean:
+ metric_type: gauge
+ alias: kafka.request.produce.time.avg
+ 99thPercentile:
+ metric_type: gauge
+ alias: kafka.request.produce.time.99percentile
+ - include:
+ domain: 'kafka.network'
+ bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Fetch'
+ attribute:
+ Mean:
+ metric_type: gauge
+ alias: kafka.request.fetch.time.avg
+ 99thPercentile:
+ metric_type: gauge
+ alias: kafka.request.fetch.time.99percentile
+ - include:
+ domain: 'kafka.network'
+ bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=UpdateMetadata'
+ attribute:
+ Mean:
+ metric_type: gauge
+ alias: kafka.request.update_metadata.time.avg
+ 99thPercentile:
+ metric_type: gauge
+ alias: kafka.request.update_metadata.time.99percentile
+ - include:
+ domain: 'kafka.network'
+ bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Metadata'
+ attribute:
+ Mean:
+ metric_type: gauge
+ alias: kafka.request.metadata.time.avg
+ 99thPercentile:
+ metric_type: gauge
+ alias: kafka.request.metadata.time.99percentile
+ - include:
+ domain: 'kafka.network'
+ bean: 'kafka.network:type=RequestMetrics,name=TotalTimeMs,request=Offsets'
+ attribute:
+ Mean:
+ metric_type: gauge
+ alias: kafka.request.offsets.time.avg
+ 99thPercentile:
+ metric_type: gauge
+ alias: kafka.request.offsets.time.99percentile
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=KafkaRequestHandlerPool,name=RequestHandlerAvgIdlePercent'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.request.handler.avg.idle.pct
+
+ #
+ # Replication stats
+ #
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=ReplicaManager,name=IsrShrinksPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.replication.isr_shrinks
+ - include:
+ domain: 'kafka.server'
+ bean: 'kafka.server:type=ReplicaManager,name=IsrExpandsPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.replication.isr_expands
+ - include:
+ domain: 'kafka.controller'
+ bean: 'kafka.controller:type=ControllerStats,name=LeaderElectionRateAndTimeMs'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.replication.leader_elections
+ - include:
+ domain: 'kafka.controller'
+ bean: 'kafka.controller:type=ControllerStats,name=UncleanLeaderElectionsPerSec'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.replication.unclean_leader_elections
+
+ #
+ # Log flush stats
+ #
+ - include:
+ domain: 'kafka.log'
+ bean: 'kafka.log:type=LogFlushStats,name=LogFlushRateAndTimeMs'
+ attribute:
+ MeanRate:
+ metric_type: gauge
+ alias: kafka.log.flush_rate
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/805fc852/deploy/images/datadog/conf.d/logstash-process.yaml
----------------------------------------------------------------------
diff --git a/deploy/images/datadog/conf.d/logstash-process.yaml b/deploy/images/datadog/conf.d/logstash-process.yaml
new file mode 100644
index 0000000..7a05002
--- /dev/null
+++ b/deploy/images/datadog/conf.d/logstash-process.yaml
@@ -0,0 +1,8 @@
+init_config:
+ pid_cache_duration: 30
+ procfs_path: /host/proc
+
+instances:
+ - name: logstash
+ search_string: ['logstash']
+ exact_match: false
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/airavata/blob/805fc852/deploy/images/datadog/conf.d/zk.yaml
----------------------------------------------------------------------
diff --git a/deploy/images/datadog/conf.d/zk.yaml b/deploy/images/datadog/conf.d/zk.yaml
new file mode 100644
index 0000000..ba07259
--- /dev/null
+++ b/deploy/images/datadog/conf.d/zk.yaml
@@ -0,0 +1,6 @@
+init_config:
+
+instances:
+ - host: localhost
+ port: 2181
+ timeout: 3
\ No newline at end of file