You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/09/10 15:03:16 UTC
[04/39] git commit: AMBARI-7197. Add rca properties to
log4j.properties from Download Configs. (aonishuk)
AMBARI-7197. Add rca properties to log4j.properties from Download Configs. (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7c8299b1
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7c8299b1
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7c8299b1
Branch: refs/heads/branch-alerts-dev
Commit: 7c8299b1c5f1885af8da1fa68a75d89d0a07a601
Parents: 8b26d8b
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Sep 8 17:25:17 2014 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Sep 8 17:25:17 2014 +0300
----------------------------------------------------------------------
.../libraries/script/script.py | 23 +++++---
.../1.3.2/hooks/before-START/scripts/params.py | 23 ++------
.../HDFS/package/scripts/hdfs_client.py | 6 ++
.../services/HDFS/package/scripts/params.py | 16 +++++-
.../MAPREDUCE/configuration/mapred-env.xml | 20 +++++++
.../MAPREDUCE/package/scripts/client.py | 7 +++
.../MAPREDUCE/package/scripts/params.py | 13 +++++
.../stacks/1.3.2/HDFS/test_hdfs_client.py | 59 ++++++++++++++++++++
.../1.3.2/MAPREDUCE/test_mapreduce_client.py | 36 +++++++++++-
.../python/stacks/1.3.2/configs/default.json | 10 +++-
10 files changed, 182 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 80bfeab..4caaa78 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -237,6 +237,20 @@ class Script(object):
"""
self.fail_with_error('configure method isn\'t implemented')
+ def generate_configs_get_template_file_content(self, filename, dicts):
+ import params
+ content = ''
+ for dict in dicts.split(','):
+ if dict.strip() in params.config['configurations']:
+ content += params.config['configurations'][dict.strip()]['content']
+
+ return content
+
+ def generate_configs_get_xml_file_content(self, filename, dict):
+ import params
+ return {'configurations':params.config['configurations'][dict],
+ 'configuration_attributes':params.config['configuration_attributes'][dict]}
+
def generate_configs(self, env):
"""
Generates config files and stores them as an archive in tmp_dir
@@ -254,17 +268,12 @@ class Script(object):
for filename, dict in file_dict.iteritems():
XmlConfig(filename,
conf_dir=conf_tmp_dir,
- configurations=params.config['configurations'][dict],
- configuration_attributes=params.config['configuration_attributes'][dict],
+ **self.generate_configs_get_xml_file_content(filename, dict)
)
for file_dict in env_configs_list:
for filename,dicts in file_dict.iteritems():
- content = ''
- for dict in dicts.split(','):
- if dict.strip() in params.config['configurations']:
- content += params.config['configurations'][dict.strip()]['content']
File(os.path.join(conf_tmp_dir, filename),
- content=InlineTemplate(content))
+ content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts)))
with closing(tarfile.open(output_filename, "w:gz")) as tar:
tar.add(conf_tmp_dir, arcname=os.path.basename("."))
tar.close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
index afbacbb..51f8d61 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/before-START/scripts/params.py
@@ -134,27 +134,12 @@ mapred_local_dir = "/tmp/hadoop-mapred/mapred/local"
dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
#log4j.properties
-rca_properties = format('''
-ambari.jobhistory.database={ambari_db_rca_url}
-ambari.jobhistory.driver={ambari_db_rca_driver}
-ambari.jobhistory.user={ambari_db_rca_username}
-ambari.jobhistory.password={ambari_db_rca_password}
-ambari.jobhistory.logger=${{hadoop.root.logger}}
+if 'mapred-env' in config['configurations'] and 'rca_properties' in config['configurations']['mapred-env']:
+ rca_properties = format(config['configurations']['mapred-env']['rca_properties'])
-log4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender
-log4j.appender.JHA.database={ambari_db_rca_url}
-log4j.appender.JHA.driver={ambari_db_rca_driver}
-log4j.appender.JHA.user={ambari_db_rca_username}
-log4j.appender.JHA.password={ambari_db_rca_password}
-
-log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA
-log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true
-
-''')
-
-if (('hdfs-log4j' in config['configurations']) and ('content' in config['configurations']['hdfs-log4j'])):
+if 'hdfs-log4j' in config['configurations']:
log4j_props = config['configurations']['hdfs-log4j']['content']
- if (('mapreduce-log4j' in config['configurations']) and ('content' in config['configurations']['mapreduce-log4j'])):
+ if 'mapreduce-log4j' in config['configurations']:
log4j_props += config['configurations']['mapreduce-log4j']['content']
if rca_enabled:
log4j_props += rca_properties
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py
index 8e3fc5d..9f783fa 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs_client.py
@@ -47,6 +47,12 @@ class HdfsClient(Script):
import params
hdfs()
+ def generate_configs_get_template_file_content(self, filename, dicts):
+ import params
+ content = super(HdfsClient,self).generate_configs_get_template_file_content(filename, dicts)
+ if filename == 'log4j.properties':
+ content += params.rca_properties
+ return content
if __name__ == "__main__":
HdfsClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py
index a7dc196..245ad92 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/params.py
@@ -173,4 +173,18 @@ ttnode_heapsize = default("/configurations/mapred-env/ttnode_heapsize","1024m")
dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
mapred_pid_dir_prefix = default("/configurations/hadoop-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
-mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
\ No newline at end of file
+mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+
+rca_enabled = False
+if 'mapred-env' in config['configurations']:
+ rca_enabled = config['configurations']['mapred-env']['rca_enabled']
+
+ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url']
+ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver']
+ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username']
+ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password']
+
+rca_properties = ''
+if rca_enabled and 'mapreduce-log4j' in config['configurations'] \
+ and 'rca_properties' in config['configurations']['mapred-env']:
+ rca_properties = format(config['configurations']['mapred-env']['rca_properties'])
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapred-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapred-env.xml b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapred-env.xml
index 95d853f..e29aab4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapred-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/configuration/mapred-env.xml
@@ -57,5 +57,25 @@
<property-type>USER</property-type>
<description>MapReduce User.</description>
</property>
+ <property>
+ <name>rca_properties</name>
+ <value>
+ambari.jobhistory.database={ambari_db_rca_url}
+ambari.jobhistory.driver={ambari_db_rca_driver}
+ambari.jobhistory.user={ambari_db_rca_username}
+ambari.jobhistory.password={ambari_db_rca_password}
+ambari.jobhistory.logger=${{hadoop.root.logger}}
+
+log4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender
+log4j.appender.JHA.database={ambari_db_rca_url}
+log4j.appender.JHA.driver={ambari_db_rca_driver}
+log4j.appender.JHA.user={ambari_db_rca_username}
+log4j.appender.JHA.password={ambari_db_rca_password}
+
+log4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA
+log4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true
+
+ </value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/client.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/client.py
index 8b98fe2..4ba1031 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/client.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/client.py
@@ -38,5 +38,12 @@ class Client(Script):
def status(self, env):
raise ClientComponentHasNoStatus()
+ def generate_configs_get_template_file_content(self, filename, dicts):
+ import params
+ content = super(Client,self).generate_configs_get_template_file_content(filename, dicts)
+ if filename == 'log4j.properties':
+ content += params.rca_properties
+ return content
+
if __name__ == "__main__":
Client().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
index 5b44cca..13642d2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/params.py
@@ -78,3 +78,16 @@ HdfsDirectory = functools.partial(
mapred_tt_group = default("/configurations/mapred-site/mapreduce.tasktracker.group", user_group)
slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+
+rca_enabled = False
+if 'mapred-env' in config['configurations']:
+ rca_enabled = config['configurations']['mapred-env']['rca_enabled']
+
+ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url']
+ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver']
+ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username']
+ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password']
+
+rca_properties = ''
+if rca_enabled and 'rca_properties' in config['configurations']['mapred-env']:
+ rca_properties = format(config['configurations']['mapred-env']['rca_properties'])
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
new file mode 100644
index 0000000..d234c5f
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from mock.mock import MagicMock, call, patch
+import tempfile
+import tarfile
+import contextlib
+from stacks.utils.RMFTestCase import *
+import os
+
+class Test(RMFTestCase):
+
+ @patch.object(tarfile,"open", new = MagicMock())
+ @patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123'))
+ @patch.object(contextlib,"closing", new = MagicMock())
+ @patch("os.path.exists", new = MagicMock(return_value=True))
+ def test_generate_configs_default(self):
+ self.executeScript("1.3.2/services/HDFS/package/scripts/hdfs_client.py",
+ classname = "HdfsClient",
+ command = "generate_configs",
+ config_file="default.json"
+ )
+ self.assertResourceCalled('Directory', '/tmp',
+ recursive = True,
+ )
+ self.printResources()
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ conf_dir = '/tmp/123',
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+ configurations = self.getConfig()['configurations']['core-site'],
+ )
+ self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
+ conf_dir = '/tmp/123',
+ configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'],
+ configurations = self.getConfig()['configurations']['mapred-site'],
+ )
+ self.assertResourceCalled('File', '/tmp/123/log4j.properties',
+ content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"),
+ )
+ self.assertResourceCalled('Directory', '/tmp/123',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
index cb88efa..2081c36 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
@@ -18,6 +18,9 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
from mock.mock import MagicMock, call, patch
+import tempfile
+import tarfile
+import contextlib
from stacks.utils.RMFTestCase import *
import os
@@ -165,4 +168,35 @@ class TestMapreduceClient(RMFTestCase):
owner = 'mapred',
group = 'hadoop',
)
- self.assertNoMoreResources()
\ No newline at end of file
+ self.assertNoMoreResources()
+
+ @patch.object(tarfile,"open", new = MagicMock())
+ @patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123'))
+ @patch.object(contextlib,"closing", new = MagicMock())
+ @patch("os.path.exists", new = MagicMock(return_value=True))
+ def test_generate_configs_default(self):
+ self.executeScript("1.3.2/services/MAPREDUCE/package/scripts/client.py",
+ classname = "Client",
+ command = "generate_configs",
+ config_file="default.json"
+ )
+ self.assertResourceCalled('Directory', '/tmp',
+ recursive = True,
+ )
+ self.assertResourceCalled('XmlConfig', 'core-site.xml',
+ conf_dir = '/tmp/123',
+ configuration_attributes = self.getConfig()['configuration_attributes']['core-site'],
+ configurations = self.getConfig()['configurations']['core-site'],
+ )
+ self.assertResourceCalled('XmlConfig', 'mapred-site.xml',
+ conf_dir = '/tmp/123',
+ configuration_attributes = self.getConfig()['configuration_attributes']['mapred-site'],
+ configurations = self.getConfig()['configurations']['mapred-site'],
+ )
+ self.assertResourceCalled('File', '/tmp/123/log4j.properties',
+ content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"),
+ )
+ self.assertResourceCalled('Directory', '/tmp/123',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/7c8299b1/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
index 3554ba1..70dee1b 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
@@ -32,8 +32,11 @@
"script": "scripts/datanode.py",
"excluded_hosts": "host1,host2",
"mark_draining_only" : "false",
- "update_exclude_file_only" : "false"
- },
+ "update_exclude_file_only" : "false",
+ "xml_configs_list":[{"core-site.xml":"core-site"},{"mapred-site.xml":"mapred-site"}],
+ "env_configs_list":[{"log4j.properties":"hdfs-log4j,mapreduce-log4j"}],
+ "output_file":"MAPREDUCE_CLIENT-configs.tar.gz"
+ },
"taskId": 18,
"public_hostname": "c6402.ambari.apache.org",
"configurations": {
@@ -326,7 +329,8 @@
"jtnode_opt_newsize": "200m",
"mapred_user": "mapred",
"hadoop_heapsize": "1024",
- "jtnode_opt_maxnewsize": "200m"
+ "jtnode_opt_maxnewsize": "200m",
+ "rca_properties": "\nambari.jobhistory.database={ambari_db_rca_url}\nambari.jobhistory.driver={ambari_db_rca_driver}\nambari.jobhistory.user={ambari_db_rca_username}\nambari.jobhistory.password={ambari_db_rca_password}\nambari.jobhistory.logger=${{hadoop.root.logger}}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database={ambari_db_rca_url}\nlog4j.appender.JHA.driver={ambari_db_rca_driver}\nlog4j.appender.JHA.user={ambari_db_rca_username}\nlog4j.appender.JHA.password={ambari_db_rca_password}\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"
},
"nagios-env": {
"hive_metastore_user_passwd": "password",