You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2014/11/06 19:39:36 UTC
[2/2] ambari git commit: AMBARI-8187. Download client configs gives
incorrect content for certain components (aonishuk)
AMBARI-8187. Download client configs gives incorrect content for certain components (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e58eda61
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e58eda61
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e58eda61
Branch: refs/heads/branch-1.7.0
Commit: e58eda61e4c94a9ffc20c6c6667fa3d183a647bf
Parents: d8d6bf4
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Nov 6 20:39:28 2014 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Nov 6 20:39:28 2014 +0200
----------------------------------------------------------------------
.../resource_management/libraries/script/script.py | 14 ++++++++++++++
.../internal/ClientConfigResourceProvider.java | 6 +++++-
.../stacks/HDP/2.0.6/services/HBASE/metainfo.xml | 5 +++++
.../stacks/HDP/2.1/services/FALCON/metainfo.xml | 4 ++--
.../test/python/stacks/1.3.2/HDFS/test_hdfs_client.py | 6 ++++++
.../stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py | 6 ++++++
.../src/test/python/stacks/1.3.2/configs/default.json | 1 +
.../test/python/stacks/2.0.6/HDFS/test_hdfs_client.py | 6 ++++++
.../src/test/python/stacks/2.0.6/configs/default.json | 1 +
9 files changed, 46 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py b/ambari-common/src/main/python/resource_management/libraries/script/script.py
index 4caaa78..001922d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/script/script.py
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -30,6 +30,7 @@ from contextlib import closing
from resource_management.libraries.resources import XmlConfig
+from resource_management.libraries.resources import PropertiesFile
from resource_management.core.resources import File, Directory
from resource_management.core.source import InlineTemplate
@@ -250,6 +251,10 @@ class Script(object):
import params
return {'configurations':params.config['configurations'][dict],
'configuration_attributes':params.config['configuration_attributes'][dict]}
+
+ def generate_configs_get_xml_file_dict(self, filename, dict):
+ import params
+ return params.config['configurations'][dict]
def generate_configs(self, env):
"""
@@ -260,6 +265,8 @@ class Script(object):
env.set_params(params)
xml_configs_list = params.config['commandParams']['xml_configs_list']
env_configs_list = params.config['commandParams']['env_configs_list']
+ properties_configs_list = params.config['commandParams']['properties_configs_list']
+
conf_tmp_dir = tempfile.mkdtemp()
output_filename = os.path.join(self.get_tmp_dir(),params.config['commandParams']['output_file'])
@@ -274,6 +281,13 @@ class Script(object):
for filename,dicts in file_dict.iteritems():
File(os.path.join(conf_tmp_dir, filename),
content=InlineTemplate(self.generate_configs_get_template_file_content(filename, dicts)))
+
+ for file_dict in properties_configs_list:
+ for filename, dict in file_dict.iteritems():
+ PropertiesFile(os.path.join(conf_tmp_dir, filename),
+ properties=self.generate_configs_get_xml_file_dict(filename, dict)
+ )
+
with closing(tarfile.open(output_filename, "w:gz")) as tar:
tar.add(conf_tmp_dir, arcname=os.path.basename("."))
tar.close()
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 339ebc4..8f59581 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -265,6 +265,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
Map<String, Object> commandParams = new HashMap<String, Object>();
List<Map<String, String>> xmlConfigs = new LinkedList<Map<String, String>>();
List<Map<String, String>> envConfigs = new LinkedList<Map<String, String>>();
+ List<Map<String, String>> propertiesConfigs = new LinkedList<Map<String, String>>();
//Fill file-dictionary configs from metainfo
for (ClientConfigFileDefinition clientConfigFile : clientConfigFiles) {
@@ -272,13 +273,16 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
fileDict.put(clientConfigFile.getFileName(), clientConfigFile.getDictionaryName());
if (clientConfigFile.getType().equals("xml")) {
xmlConfigs.add(fileDict);
- } else {
+ } else if (clientConfigFile.getType().equals("env")) {
envConfigs.add(fileDict);
+ } else if (clientConfigFile.getType().equals("properties")) {
+ propertiesConfigs.add(fileDict);
}
}
commandParams.put("xml_configs_list", xmlConfigs);
commandParams.put("env_configs_list", envConfigs);
+ commandParams.put("properties_configs_list", propertiesConfigs);
commandParams.put("output_file", componentName + "-configs.tar.gz");
Map<String, Object> jsonContent = new TreeMap<String, Object>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
index 0816591..fd290df 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/metainfo.xml
@@ -97,6 +97,11 @@
<dictionaryName>hbase-env</dictionaryName>
</configFile>
<configFile>
+ <type>xml</type>
+ <fileName>hbase-policy.xml</fileName>
+ <dictionaryName>hbase-policy</dictionaryName>
+ </configFile>
+ <configFile>
<type>env</type>
<fileName>log4j.properties</fileName>
<dictionaryName>hbase-log4j</dictionaryName>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
index b4e6883..78336e6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/FALCON/metainfo.xml
@@ -41,12 +41,12 @@
<dictionaryName>falcon-env</dictionaryName>
</configFile>
<configFile>
- <type>xml</type>
+ <type>properties</type>
<fileName>runtime.properties</fileName>
<dictionaryName>falcon-runtime.properties</dictionaryName>
</configFile>
<configFile>
- <type>xml</type>
+ <type>properties</type>
<fileName>startup.properties</fileName>
<dictionaryName>falcon-startup.properties</dictionaryName>
</configFile>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
index 80da9b6..f670ac4 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_hdfs_client.py
@@ -52,6 +52,12 @@ class Test(RMFTestCase):
self.assertResourceCalled('File', '/tmp/123/log4j.properties',
content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"),
)
+ self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
+ properties = UnknownConfigurationMock(),
+ )
+ self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
+ properties = UnknownConfigurationMock(),
+ )
self.assertResourceCalled('Directory', '/tmp/123',
action = ['delete'],
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
index 2081c36..1621b42 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/MAPREDUCE/test_mapreduce_client.py
@@ -196,6 +196,12 @@ class TestMapreduceClient(RMFTestCase):
self.assertResourceCalled('File', '/tmp/123/log4j.properties',
content = InlineTemplate("log4jproperties\nline2log4jproperties\nline2\nambari.jobhistory.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nambari.jobhistory.driver=org.postgresql.Driver\nambari.jobhistory.user=mapred\nambari.jobhistory.password=mapred\nambari.jobhistory.logger=${hadoop.root.logger}\n\nlog4j.appender.JHA=org.apache.ambari.log4j.hadoop.mapreduce.jobhistory.JobHistoryAppender\nlog4j.appender.JHA.database=jdbc:postgresql://c6401.ambari.apache.org/ambarirca\nlog4j.appender.JHA.driver=org.postgresql.Driver\nlog4j.appender.JHA.user=mapred\nlog4j.appender.JHA.password=mapred\n\nlog4j.logger.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=DEBUG,JHA\nlog4j.additivity.org.apache.hadoop.mapred.JobHistory$JobHistoryLogger=true\n\n"),
)
+ self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
+ properties = UnknownConfigurationMock(),
+ )
+ self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
+ properties = UnknownConfigurationMock(),
+ )
self.assertResourceCalled('Directory', '/tmp/123',
action = ['delete'],
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
index 0256444..a39d51d 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/1.3.2/configs/default.json
@@ -35,6 +35,7 @@
"update_exclude_file_only" : "false",
"xml_configs_list":[{"core-site.xml":"core-site"},{"mapred-site.xml":"mapred-site"}],
"env_configs_list":[{"log4j.properties":"hdfs-log4j,mapreduce-log4j"}],
+ "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
"output_file":"MAPREDUCE_CLIENT-configs.tar.gz"
},
"taskId": 18,
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
index 0a4bfb5..7c9e42b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -52,6 +52,12 @@ class Test(RMFTestCase):
content = InlineTemplate(self.getConfig()['configurations']['hdfs-log4j']['content']+
self.getConfig()['configurations']['yarn-log4j']['content']),
)
+ self.assertResourceCalled('PropertiesFile', '/tmp/123/runtime.properties',
+ properties = UnknownConfigurationMock(),
+ )
+ self.assertResourceCalled('PropertiesFile', '/tmp/123/startup.properties',
+ properties = UnknownConfigurationMock(),
+ )
self.assertResourceCalled('Directory', '/tmp/123',
action = ['delete'],
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/e58eda61/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 2dc4a8d..6b68e0b 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -32,6 +32,7 @@
"update_exclude_file_only" : "false",
"xml_configs_list":[{"hdfs-site.xml":"hdfs-site"}],
"env_configs_list":[{"hadoop-env.sh":"hadoop-env"},{"log4j.properties":"hdfs-log4j,yarn-log4j"}],
+ "properties_configs_list":[{"runtime.properties":"falcon-runtime.properties"},{"startup.properties":"falcon-startup.properties"}],
"output_file":"HDFS_CLIENT-configs.tar.gz"
},
"taskId": 152,