You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@senssoft.apache.org by ms...@apache.org on 2018/06/04 22:10:32 UTC
incubator-senssoft git commit: Added support for monitoring kafka,
zookeeper, and updated config for postgresql to monitor TAPs database.
Repository: incubator-senssoft
Updated Branches:
refs/heads/ansible ae3ebdfd2 -> c4b63e90b
Added support for monitoring kafka, zookeeper, and updated config for postgresql to monitor TAPs database.
Project: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/commit/c4b63e90
Tree: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/tree/c4b63e90
Diff: http://git-wip-us.apache.org/repos/asf/incubator-senssoft/diff/c4b63e90
Branch: refs/heads/ansible
Commit: c4b63e90b1bda7807490d33b0cde19726831f786
Parents: ae3ebdf
Author: msbeard <ms...@apache.org>
Authored: Mon Jun 4 18:10:20 2018 -0400
Committer: msbeard <ms...@apache.org>
Committed: Mon Jun 4 18:10:20 2018 -0400
----------------------------------------------------------------------
.gitignore | 2 +
docker/beats/metricbeat/metricbeat.yml | 2 +-
docker/beats/metricbeat/modules.d/kafka.yml | 9 ++++
docker/beats/metricbeat/modules.d/kibana.yml | 2 +-
.../beats/metricbeat/modules.d/kubernetes.yml | 47 ++++++++++----------
.../beats/metricbeat/modules.d/postgresql.yml | 22 +++------
docker/beats/metricbeat/modules.d/zookeeper.yml | 6 +++
docker/docker-compose.yml | 8 +++-
8 files changed, 56 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index ba433d6..c8facd7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,5 @@
+env
+venv
.idea
release/incubator-*
release/apache-*
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/metricbeat.yml
----------------------------------------------------------------------
diff --git a/docker/beats/metricbeat/metricbeat.yml b/docker/beats/metricbeat/metricbeat.yml
index 3c61d77..a987a43 100644
--- a/docker/beats/metricbeat/metricbeat.yml
+++ b/docker/beats/metricbeat/metricbeat.yml
@@ -4,7 +4,7 @@ metricbeat.config.modules:
path: ${path.config}/modules.d/*.yml
# Period on which files under path should be checked for changes
- reload.period: 10s
+ reload.period: 30s
# Set to true to enable config reloading
reload.enabled: true
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/kafka.yml
----------------------------------------------------------------------
diff --git a/docker/beats/metricbeat/modules.d/kafka.yml b/docker/beats/metricbeat/modules.d/kafka.yml
new file mode 100644
index 0000000..b1954cc
--- /dev/null
+++ b/docker/beats/metricbeat/modules.d/kafka.yml
@@ -0,0 +1,9 @@
+#---------------------------- Kafka Module ---------------------------
+- module: kafka
+ metricsets: ["consumergroup", "partition"]
+ period: 10s
+ hosts: ["${KAFKA_URL}"]
+ enabled: false
+
+ # List of Topics to query metadata for. If empty, all topics will be queried.
+ #topics: []
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/kibana.yml
----------------------------------------------------------------------
diff --git a/docker/beats/metricbeat/modules.d/kibana.yml b/docker/beats/metricbeat/modules.d/kibana.yml
index 2134a62..0d83b05 100644
--- a/docker/beats/metricbeat/modules.d/kibana.yml
+++ b/docker/beats/metricbeat/modules.d/kibana.yml
@@ -1,6 +1,6 @@
#------------------------------- Kibana Module -------------------------------
- module: kibana
metricsets: ["status"]
- period: 10s
+ period: 30s
enabled: true
hosts: ["${KIBANA_URL}"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/kubernetes.yml
----------------------------------------------------------------------
diff --git a/docker/beats/metricbeat/modules.d/kubernetes.yml b/docker/beats/metricbeat/modules.d/kubernetes.yml
index b3e018d..f58ea52 100644
--- a/docker/beats/metricbeat/modules.d/kubernetes.yml
+++ b/docker/beats/metricbeat/modules.d/kubernetes.yml
@@ -1,29 +1,30 @@
#----------------------------- Kubernetes Module -----------------------------
# Node metrics, from kubelet:
-#- module: kubernetes
-# metricsets:
-# - node
-# - system
-# - pod
-# - container
-# - volume
-# period: 10s
-# hosts: ["localhost:10255"]
+- module: kubernetes
+ enabled: false
+ metricsets:
+ - node
+ - system
+ - pod
+ - container
+ - volume
+ period: 10s
+ hosts: ["localhost:10255"]
# State metrics from kube-state-metrics service:
-#- module: kubernetes
-# enabled: false
-# metricsets:
-# - state_node
-# - state_deployment
-# - state_replicaset
-# - state_pod
-# - state_container
-# period: 10s
-# hosts: ["kube-state-metrics:8080"]
+- module: kubernetes
+ enabled: false
+ metricsets:
+ - state_node
+ - state_deployment
+ - state_replicaset
+ - state_pod
+ - state_container
+ period: 10s
+ hosts: ["kube-state-metrics:8080"]
# Kubernetes events
-#- module: kubernetes
-# enabled: false
-# metricsets:
-# - event
\ No newline at end of file
+- module: kubernetes
+ enabled: false
+ metricsets:
+ - event
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/postgresql.yml
----------------------------------------------------------------------
diff --git a/docker/beats/metricbeat/modules.d/postgresql.yml b/docker/beats/metricbeat/modules.d/postgresql.yml
index 2b02360..363fe4b 100644
--- a/docker/beats/metricbeat/modules.d/postgresql.yml
+++ b/docker/beats/metricbeat/modules.d/postgresql.yml
@@ -1,25 +1,17 @@
#----------------------------- PostgreSQL Module -----------------------------
-#- module: postgresql
-# metricsets:
-# # Stats about every PostgreSQL database
-# - database
-#
-# # Stats about the background writer process's activity
-# - bgwriter
-#
-# # Stats about every PostgreSQL process
-# - activity
-#
-# period: 10s
+- module: postgresql
+ metricsets: ["database", "activity"]
+ period: 10s
+ enabled: true
# The host must be passed as PostgreSQL URL. Example:
# postgres://localhost:5432?sslmode=disable
# The available parameters are documented here:
# https://godoc.org/github.com/lib/pq#hdr-Connection_String_Parameters
-# hosts: ["postgres://localhost:5432"]
+ hosts: ["${POSTGRESQL_URL}"]
# Username to use when connecting to PostgreSQL. Empty by default.
- #username: user
+ username: "${POSTGRESQL_USERNAME}"
# Password to use when connecting to PostgreSQL. Empty by default.
- #password: pass
\ No newline at end of file
+ password: "${POSTGRESQL_PASSWORD}"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/beats/metricbeat/modules.d/zookeeper.yml
----------------------------------------------------------------------
diff --git a/docker/beats/metricbeat/modules.d/zookeeper.yml b/docker/beats/metricbeat/modules.d/zookeeper.yml
new file mode 100644
index 0000000..ca67993
--- /dev/null
+++ b/docker/beats/metricbeat/modules.d/zookeeper.yml
@@ -0,0 +1,6 @@
+#---------------------------- ZooKeeper Module ---------------------------
+- module: zookeeper
+ enabled: false
+ metricsets: ["mntr"]
+ period: 10s
+ hosts: ["${ZOOKEEPER_URL}"]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-senssoft/blob/c4b63e90/docker/docker-compose.yml
----------------------------------------------------------------------
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 30507c6..316b7f5 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -123,7 +123,7 @@ services:
# Specific container and configuration to monitor the HOST (filesystem, memory, processes,...)
metricbeat:
image: docker.elastic.co/beats/metricbeat:6.2.4
- restart: unless-stopped
+# restart: unless-stopped
user: root
pid: host
volumes:
@@ -133,7 +133,7 @@ services:
- /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
#We mount the host filesystem so we can report on disk usage with the system module
- /:/hostfs:ro
- #Mount the metricbeat configuration so users can make edit
+ #Mount the metricbeat configuration so users can make edits
- ./beats/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml
#Mount the modules.d directory into the container. This allows user to
#potentially make changes to the modules and they will be dynamically loaded.
@@ -146,10 +146,14 @@ services:
- ELASTICSEARCH_URL=http://elasticsearch:9200
- KIBANA_URL=http://kibana:5601
- LOGSTASH_URL=http://logstash:9600
+ - POSTGRESQL_URL="postgresql://db:5432/tapdb?sslmode=disable"
+ - POSTGRESQL_USERNAME=tapuser
+ - POSTGRESQL_PASSWORD="Dr@p3rUs3r"
extra_hosts:
- "elasticsearch:172.17.0.1" # The IP of docker0 interface to access host from container
- "kibana:172.17.0.1" # The IP of docker0 interface to access host from container
- "logstash:172.17.0.1"
+ - "db:172.17.0.1"
command: metricbeat -e -system.hostfs=/hostfs -strict.perms=false
network_mode: host # Mandatory to monitor HOST filesystem, memory, processes,...