You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluo.apache.org by ar...@apache.org on 2019/11/22 23:37:33 UTC
[fluo-muchos] branch master updated: Updates to Azure Log Analytics
(OMS) integration (#299)
This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 91e3d8c Updates to Azure Log Analytics (OMS) integration (#299)
91e3d8c is described below
commit 91e3d8c9bd730265449996e8d71f1e22606c0bc8
Author: Arvind Shyamsundar <ar...@apache.org>
AuthorDate: Fri Nov 22 15:37:24 2019 -0800
Updates to Azure Log Analytics (OMS) integration (#299)
* Updates to Azure Log Analytics (OMS) integration
- Align with current upstream Azure OMS Agent environment variable names
- Fixed a dormant bug related to validating boolean vars in muchos.props
- Send Accumulo metrics to statsd (for Azure Log Analytics) in addition to InfluxDB
- Add Hadoop metrics (currently sent only to statsd for Azure Log Analytics)
* Address review comments
- Moved the metrics2 file to the correct Ansible templates folder
- Also updated the Ansible Hadoop templates task to copy this file
---
.../templates/hadoop-metrics2-accumulo.properties | 10 ++++++++++
ansible/roles/common/tasks/azure_oms.yml | 2 +-
ansible/roles/hadoop/tasks/main.yml | 1 +
.../templates/hadoop-metrics2.properties} | 17 ++++++++++++-----
lib/muchos/config/decorators.py | 2 +-
5 files changed, 25 insertions(+), 7 deletions(-)
diff --git a/ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties b/ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties
index 5e1f9d0..9b09b0e 100644
--- a/ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties
+++ b/ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties
@@ -16,7 +16,17 @@
#
*.period=30
+{% if 'metrics' in groups %}
accumulo.sink.graphite.class=org.apache.hadoop.metrics2.sink.GraphiteSink
accumulo.sink.graphite.server_host={{ groups['metrics'][0] }}
accumulo.sink.graphite.server_port=2004
accumulo.sink.graphite.metrics_prefix=accumulo
+{% endif %}
+
+{% if cluster_type == 'azure' %}
+*.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+accumulo.sink.statsd.server.host=127.0.0.1
+accumulo.sink.statsd.server.port=8125
+accumulo.sink.statsd.skip.hostname=true
+accumulo.sink.statsd.service.name=master
+{% endif %}
diff --git a/ansible/roles/common/tasks/azure_oms.yml b/ansible/roles/common/tasks/azure_oms.yml
index c94cf9a..132d5ab 100644
--- a/ansible/roles/common/tasks/azure_oms.yml
+++ b/ansible/roles/common/tasks/azure_oms.yml
@@ -21,7 +21,7 @@
template: src=../templates/etc/collectd.d/azure.conf.j2 dest=/etc/collectd.d/azure.conf
- name: "Ensure OMS Agent is installed"
- shell: eval $(curl -s https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh | grep -e 'GITHUB_RELEASE=' -e 'BUNDLE_X64=') && wget ${GITHUB_RELEASE}${BUNDLE_X64} && sh ./${BUNDLE_X64} --upgrade
+ shell: eval $(curl -s https://raw.githubusercontent.com/Microsoft/OMS-Agent-for-Linux/master/installer/scripts/onboard_agent.sh | grep -e 'GITHUB_RELEASE_X64=' -e 'BUNDLE_X64=') && wget ${GITHUB_RELEASE_X64}${BUNDLE_X64} && sh ./${BUNDLE_X64} --upgrade
args:
creates: /opt/microsoft/omsagent/bin/omsadmin.sh
register: omsagent_installed
diff --git a/ansible/roles/hadoop/tasks/main.yml b/ansible/roles/hadoop/tasks/main.yml
index 36c7a62..d0219b3 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -24,6 +24,7 @@
- hdfs-site.xml
- yarn-site.xml
- mapred-site.xml
+ - hadoop-metrics2.properties
- name: "configure hadoop 2"
template: src={{ item }} dest={{ hadoop_home }}/etc/hadoop/{{ item }}
with_items:
diff --git a/ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties b/ansible/roles/hadoop/templates/hadoop-metrics2.properties
similarity index 63%
copy from ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties
copy to ansible/roles/hadoop/templates/hadoop-metrics2.properties
index 5e1f9d0..40b4e66 100644
--- a/ansible/roles/accumulo/templates/hadoop-metrics2-accumulo.properties
+++ b/ansible/roles/hadoop/templates/hadoop-metrics2.properties
@@ -15,8 +15,15 @@
# limitations under the License.
#
-*.period=30
-accumulo.sink.graphite.class=org.apache.hadoop.metrics2.sink.GraphiteSink
-accumulo.sink.graphite.server_host={{ groups['metrics'][0] }}
-accumulo.sink.graphite.server_port=2004
-accumulo.sink.graphite.metrics_prefix=accumulo
+{% if cluster_type == 'azure' %}
+*.sink.statsd.class=org.apache.hadoop.metrics2.sink.StatsDSink
+*.sink.statsd.period=60
+namenode.sink.statsd.server.host=127.0.0.1
+namenode.sink.statsd.server.port=8125
+namenode.sink.statsd.skip.hostname=true
+namenode.sink.statsd.service.name=NameNode
+datanode.sink.statsd.server.host=127.0.0.1
+datanode.sink.statsd.server.port=8125
+datanode.sink.statsd.skip.hostname=true
+datanode.sink.statsd.service.name=DataNode
+{% endif %}
diff --git a/lib/muchos/config/decorators.py b/lib/muchos/config/decorators.py
index 57151b8..c3cf8d2 100644
--- a/lib/muchos/config/decorators.py
+++ b/lib/muchos/config/decorators.py
@@ -74,7 +74,7 @@ def default(val):
except:
return val
else:
- if res in [None, 0, ''] or len(res) == 0:
+ if res in [None, 0, ''] or isinstance(res, str) and len(res) == 0:
return val
return res
return wrapper