You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ds...@apache.org on 2018/05/04 11:04:19 UTC
[ambari] branch trunk updated: [AMBARI-23751] Ambari ViewFS: Add
ability to add XML inclusion node to a configuration file (dsen) (#1168)
This is an automated email from the ASF dual-hosted git repository.
dsen pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6bcefac [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) (#1168)
6bcefac is described below
commit 6bcefac7720077204057f97bd5571db7529292da
Author: Dmitry Sen <ds...@apache.org>
AuthorDate: Fri May 4 14:04:17 2018 +0300
[AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) (#1168)
* [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen)
* [AMBARI-23751] Ambari ViewFS: Add ability to add XML inclusion node to a configuration file (dsen) - UT fix and hooks change
---
.../resource_management/TestXmlConfigResource.py | 36 +++++++++++++++++++---
.../libraries/providers/xml_config.py | 10 ++++--
.../libraries/resources/xml_config.py | 1 +
.../stack-hooks/after-INSTALL/scripts/params.py | 10 ++++++
.../after-INSTALL/scripts/shared_initialization.py | 11 ++++++-
.../hooks/after-INSTALL/test_after_install.py | 12 +++++---
6 files changed, 68 insertions(+), 12 deletions(-)
diff --git a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
index 09eafc9..9b4c1d5 100644
--- a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
@@ -68,7 +68,7 @@ class TestXmlConfigResource(TestCase):
configuration_attributes={}
)
- create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n </configuration>', encoding='UTF-8')
+ create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n </configuration>', encoding='UTF-8')
@patch("resource_management.core.providers.system._ensure_metadata")
@@ -97,8 +97,36 @@ class TestXmlConfigResource(TestCase):
configuration_attributes={'attr': {'property1': 'attr_value'}}
)
- create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>', encoding='UTF-8')
+ create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>', encoding='UTF-8')
+ @patch("resource_management.core.providers.system._ensure_metadata")
+ @patch("resource_management.core.sudo.create_file")
+ @patch("resource_management.core.sudo.path_exists")
+ @patch("resource_management.core.sudo.path_isdir")
+ @patch.object(time, "asctime")
+ def test_action_create_simple_xml_config_with_inclusion(self,
+ time_asctime_mock,
+ os_path_isdir_mock,
+ os_path_exists_mock,
+ create_file_mock,
+ ensure_mock):
+ """
+ Tests if 'create' action - creates new non existent xml file and write proper data
+ where configurations={"Some conf":"Some value"}
+ """
+ os_path_isdir_mock.side_effect = [False, True]
+ os_path_exists_mock.return_value = False
+ time_asctime_mock.return_value = 'Wed 2014-02'
+
+ with Environment('/') as env:
+ XmlConfig('file.xml',
+ conf_dir='/dir/conf',
+ configurations={'property1': 'value1'},
+ configuration_attributes={'attr': {'property1': 'attr_value'}},
+ xml_include_file="/dif/conf/include_file.xml"
+ )
+
+ create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n <xi:include href="/dif/conf/include_file.xml"/>\n \n </configuration>', encoding='UTF-8')
@patch("resource_management.core.providers.system._ensure_metadata")
@patch("resource_management.core.sudo.create_file")
@@ -150,7 +178,7 @@ class TestXmlConfigResource(TestCase):
}
})
- create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L [...]
+ create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.3</n [...]
@patch("resource_management.core.providers.system._ensure_metadata")
@patch("resource_management.core.sudo.create_file")
@@ -183,7 +211,7 @@ class TestXmlConfigResource(TestCase):
configuration_attributes={}
)
- create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be third</value>\n </property>\n \n <prop [...]
+ create_file_mock.assert_called_with('/dir/conf/file.xml', u' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>first</name>\n <value>should be first</value>\n </property>\n \n <property>\n <name>second</name>\n <value>should be second</value>\n </property>\n \n <property>\n <name>third</name>\n <value>should be thi [...]
@patch("resource_management.libraries.providers.xml_config.File")
@patch("resource_management.core.sudo.path_exists")
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
index eebd0dd..bf196b5 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/xml_config.py
@@ -38,7 +38,7 @@ class XmlConfigProvider(Provider):
configuration_attrs = {} if is_empty(self.resource.configuration_attributes) else self.resource.configuration_attributes
# |e - for html-like escaping of <,>,',"
- config_content = InlineTemplate(''' <configuration>
+ config_content = InlineTemplate(''' <configuration xmlns:xi="http://www.w3.org/2001/XInclude">
{% for key, value in configurations_dict|dictsort %}
<property>
<name>{{ key|e }}</name>
@@ -54,8 +54,12 @@ class XmlConfigProvider(Provider):
{%- endif %}
</property>
{% endfor %}
- </configuration>''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source], configurations_dict=self.resource.configurations,
- configuration_attrs=configuration_attrs)
+ {%- if not xml_include_file is none %}
+ <xi:include href="{{xml_include_file|e}}"/>
+ {% endif %}
+ </configuration>''', extra_imports=[time, resource_management, resource_management.core, resource_management.core.source],
+ configurations_dict=self.resource.configurations,
+ configuration_attrs=configuration_attrs, xml_include_file=self.resource.xml_include_file)
xml_config_dest_file_path = os.path.join(xml_config_provider_config_dir, filename)
Logger.info("Generating config: {0}".format(xml_config_dest_file_path))
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py
index 882d6d5..c46f35c 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/xml_config.py
@@ -34,6 +34,7 @@ class XmlConfig(Resource):
mode = ResourceArgument()
owner = ResourceArgument()
group = ResourceArgument()
+ xml_include_file = ResourceArgument()
encoding = ResourceArgument(default="UTF-8")
diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
index 91c75dd..e66d6ac 100644
--- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
+++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/params.py
@@ -102,6 +102,16 @@ has_namenode = not len(namenode_host) == 0
if has_namenode or dfs_type == 'HCFS':
hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+ mount_table_xml_inclusion_file_full_path = None
+ mount_table_content = None
+ if 'mount-table' in config['configurations']:
+ xml_inclusion_file_name = 'mount-table.xml'
+ mount_table = config['configurations']['mount-table']
+
+ if 'content' in mount_table and mount_table['content'].strip():
+ mount_table_xml_inclusion_file_full_path = os.path.join(hadoop_conf_dir, xml_inclusion_file_name)
+ mount_table_content = mount_table['content']
+
link_configs_lock_file = get_config_lock_file()
stack_select_lock_file = os.path.join(tmp_dir, "stack_select_lock_file")
diff --git a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py
index 22543bc..d82f565 100644
--- a/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stack-hooks/after-INSTALL/scripts/shared_initialization.py
@@ -85,7 +85,16 @@ def setup_config():
configuration_attributes=params.config['configurationAttributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group,
- only_if=format("ls {hadoop_conf_dir}"))
+ only_if=format("ls {hadoop_conf_dir}"),
+ xml_include_file=params.mount_table_xml_inclusion_file_full_path
+ )
+
+ if params.mount_table_content:
+ File(os.path.join(params.hadoop_conf_dir, params.xml_inclusion_file_name),
+ owner=params.hdfs_user,
+ group=params.user_group,
+ content=params.mount_table_content
+ )
Directory(params.logsearch_logfeeder_conf,
mode=0755,
diff --git a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
index 0791784..c03a10f 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/hooks/after-INSTALL/test_after_install.py
@@ -57,7 +57,8 @@ class TestHookAfterInstall(RMFTestCase):
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configurationAttributes']['core-site'],
- only_if="ls /etc/hadoop/conf")
+ only_if="ls /etc/hadoop/conf",
+ xml_include_file=None)
self.assertResourceCalled('Directory',
'/usr/lib/ambari-logsearch-logfeeder/conf',
mode = 0755,
@@ -104,7 +105,8 @@ class TestHookAfterInstall(RMFTestCase):
conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf",
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configurationAttributes']['core-site'],
- only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf")
+ only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf",
+ xml_include_file=None)
self.assertResourceCalled('Directory',
'/usr/lib/ambari-logsearch-logfeeder/conf',
@@ -172,7 +174,8 @@ class TestHookAfterInstall(RMFTestCase):
conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf",
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configurationAttributes']['core-site'],
- only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf")
+ only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf",
+ xml_include_file=None)
self.assertResourceCalled('Directory',
'/usr/lib/ambari-logsearch-logfeeder/conf',
@@ -275,7 +278,8 @@ class TestHookAfterInstall(RMFTestCase):
conf_dir = "/usr/hdp/2.3.0.0-1234/hadoop/conf",
configurations = self.getConfig()['configurations']['core-site'],
configuration_attributes = self.getConfig()['configurationAttributes']['core-site'],
- only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf")
+ only_if="ls /usr/hdp/2.3.0.0-1234/hadoop/conf",
+ xml_include_file=None)
self.assertResourceCalled('Directory',
'/usr/lib/ambari-logsearch-logfeeder/conf',
--
To stop receiving notification emails like this one, please contact
dsen@apache.org.