You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/07/18 20:16:25 UTC
[3/3] git commit: AMBARI-6531. Config attributes should be propagated
to and persisted by Ambari agents.
AMBARI-6531. Config attributes should be propagated to and persisted by Ambari agents.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a1e9fa9a
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a1e9fa9a
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a1e9fa9a
Branch: refs/heads/trunk
Commit: a1e9fa9a2442f6d3ff89c831a7c1f5821bc4e673
Parents: 13464ca
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Thu Jul 17 15:56:38 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Fri Jul 18 09:48:39 2014 -0700
----------------------------------------------------------------------
.../libraries/providers/xml_config.py | 12 +-
.../libraries/resources/xml_config.py | 1 +
.../TestXmlConfigResource.py | 37 +++-
.../actionmanager/ExecutionCommandWrapper.java | 16 ++
.../ambari/server/agent/ExecutionCommand.java | 13 ++
.../ambari/server/agent/HeartbeatMonitor.java | 13 +-
.../ambari/server/agent/StatusCommand.java | 13 ++
.../controller/AmbariActionExecutionHelper.java | 2 +
.../AmbariCustomCommandExecutionHelper.java | 6 +
.../AmbariManagementControllerImpl.java | 5 +-
.../ambari/server/state/ConfigHelper.java | 66 ++++++-
.../apache/ambari/server/utils/StageUtils.java | 8 +
.../scripts/shared_initialization.py | 1 +
.../services/HBASE/package/scripts/hbase.py | 4 +
.../1.3.2/services/HDFS/package/scripts/hdfs.py | 2 +
.../1.3.2/services/HIVE/package/scripts/hcat.py | 1 +
.../1.3.2/services/HIVE/package/scripts/hive.py | 1 +
.../MAPREDUCE/package/scripts/mapreduce.py | 3 +
.../services/OOZIE/package/scripts/oozie.py | 1 +
.../services/WEBHCAT/package/scripts/webhcat.py | 1 +
.../scripts/shared_initialization.py | 1 +
.../services/HBASE/package/scripts/hbase.py | 4 +
.../2.0.6/services/HDFS/package/scripts/hdfs.py | 2 +
.../2.0.6/services/HIVE/package/scripts/hcat.py | 1 +
.../2.0.6/services/HIVE/package/scripts/hive.py | 2 +
.../services/OOZIE/package/scripts/oozie.py | 1 +
.../services/WEBHCAT/package/scripts/webhcat.py | 1 +
.../2.0.6/services/YARN/package/scripts/yarn.py | 7 +
.../services/TEZ/package/scripts/tez.py | 1 +
.../services/YARN/package/scripts/yarn.py | 4 +
.../HDP/2.1/services/TEZ/package/scripts/tez.py | 1 +
.../AmbariManagementControllerTest.java | 2 +
.../ambari/server/state/ConfigHelperTest.java | 173 +++++++++++++++++++
.../src/test/python/TestResourceFilesKeeper.py | 2 +-
.../stacks/1.3.2/HBASE/test_hbase_client.py | 18 +-
.../stacks/1.3.2/HBASE/test_hbase_master.py | 18 +-
.../1.3.2/HBASE/test_hbase_regionserver.py | 18 +-
.../python/stacks/1.3.2/HDFS/test_datanode.py | 2 +
.../python/stacks/1.3.2/HDFS/test_namenode.py | 2 +
.../python/stacks/1.3.2/HDFS/test_snamenode.py | 2 +
.../stacks/1.3.2/HIVE/test_hcat_client.py | 2 +
.../stacks/1.3.2/HIVE/test_hive_client.py | 2 +
.../stacks/1.3.2/HIVE/test_hive_metastore.py | 2 +
.../stacks/1.3.2/HIVE/test_hive_server.py | 2 +
.../1.3.2/MAPREDUCE/test_mapreduce_client.py | 2 +
.../MAPREDUCE/test_mapreduce_historyserver.py | 2 +
.../MAPREDUCE/test_mapreduce_jobtracker.py | 2 +
.../MAPREDUCE/test_mapreduce_tasktracker.py | 2 +
.../stacks/1.3.2/OOZIE/test_oozie_client.py | 2 +
.../stacks/1.3.2/OOZIE/test_oozie_server.py | 4 +
.../stacks/1.3.2/WEBHCAT/test_webhcat_server.py | 6 +-
.../1.3.2/configs/default.hbasedecom.json | 49 ++++++
.../python/stacks/1.3.2/configs/default.json | 49 ++++++
.../1.3.2/configs/default.non_gmetad_host.json | 51 +++++-
.../python/stacks/1.3.2/configs/secured.json | 51 +++++-
.../1.3.2/configs/secured_no_jce_name.json | 51 +++++-
.../hooks/after-INSTALL/test_after_install.py | 1 +
.../stacks/2.0.6/HBASE/test_hbase_client.py | 18 +-
.../stacks/2.0.6/HBASE/test_hbase_master.py | 18 +-
.../2.0.6/HBASE/test_hbase_regionserver.py | 18 +-
.../python/stacks/2.0.6/HDFS/test_datanode.py | 2 +
.../stacks/2.0.6/HDFS/test_journalnode.py | 2 +
.../python/stacks/2.0.6/HDFS/test_namenode.py | 2 +
.../python/stacks/2.0.6/HDFS/test_snamenode.py | 2 +
.../test/python/stacks/2.0.6/HDFS/test_zkfc.py | 2 +
.../stacks/2.0.6/HIVE/test_hcat_client.py | 2 +
.../stacks/2.0.6/HIVE/test_hive_client.py | 4 +
.../stacks/2.0.6/HIVE/test_hive_metastore.py | 4 +
.../stacks/2.0.6/HIVE/test_hive_server.py | 4 +
.../stacks/2.0.6/OOZIE/test_oozie_client.py | 4 +
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 4 +
.../stacks/2.0.6/WEBHCAT/test_webhcat_server.py | 6 +-
.../stacks/2.0.6/YARN/test_historyserver.py | 12 ++
.../stacks/2.0.6/YARN/test_mapreduce2_client.py | 12 ++
.../stacks/2.0.6/YARN/test_nodemanager.py | 12 ++
.../stacks/2.0.6/YARN/test_resourcemanager.py | 12 ++
.../stacks/2.0.6/YARN/test_yarn_client.py | 18 ++
.../2.0.6/configs/default.hbasedecom.json | 51 +++++-
.../python/stacks/2.0.6/configs/default.json | 66 +++++++
.../2.0.6/configs/default.non_gmetad_host.json | 49 ++++++
.../stacks/2.0.6/configs/flume_target.json | 49 ++++++
.../python/stacks/2.0.6/configs/ha_default.json | 51 +++++-
.../python/stacks/2.0.6/configs/ha_secured.json | 52 +++++-
.../python/stacks/2.0.6/configs/secured.json | 68 +++++++-
.../2.0.6/configs/secured_no_jce_name.json | 51 +++++-
.../hooks/after-INSTALL/test_after_install.py | 1 +
.../stacks/2.1/HIVE/test_hive_metastore.py | 4 +
.../python/stacks/2.1/TEZ/test_tez_client.py | 1 +
.../stacks/2.1/YARN/test_apptimelineserver.py | 6 +
.../test/python/stacks/2.1/configs/default.json | 66 +++++++
.../test/python/stacks/2.1/configs/secured.json | 75 +++++++-
.../dummy_stack/HIVE/package/scripts/hive.py | 1 +
.../2.0.7/services/HIVE/package/scripts/hive.py | 1 +
93 files changed, 1433 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-agent/src/main/python/resource_management/libraries/providers/xml_config.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/xml_config.py b/ambari-agent/src/main/python/resource_management/libraries/providers/xml_config.py
index 84b07ff..b2c6acb 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/providers/xml_config.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/providers/xml_config.py
@@ -35,9 +35,19 @@ class XmlConfigProvider(Provider):
<property>
<name>{{ key|e }}</name>
<value>{{ value|e }}</value>
+ {%- if not configuration_attrs is none -%}
+ {%- for attrib_name, attrib_occurances in configuration_attrs.items() -%}
+ {%- for property_name, attrib_value in attrib_occurances.items() -%}
+ {% if property_name == key and attrib_name %}
+ <{{attrib_name|e}}>{{attrib_value|e}}</{{attrib_name|e}}>
+ {%- endif -%}
+ {%- endfor -%}
+ {%- endfor -%}
+ {%- endif %}
</property>
{% endfor %}
- </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations)
+ </configuration>''', extra_imports=[time], configurations_dict=self.resource.configurations,
+ configuration_attrs=self.resource.configuration_attributes)
Logger.info(format("Generating config: {conf_dir}/{filename}"))
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-agent/src/main/python/resource_management/libraries/resources/xml_config.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/resources/xml_config.py b/ambari-agent/src/main/python/resource_management/libraries/resources/xml_config.py
index 2b8ff6f..37eb511 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/resources/xml_config.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/resources/xml_config.py
@@ -28,6 +28,7 @@ class XmlConfig(Resource):
filename = ResourceArgument(default=lambda obj: obj.name)
configurations = ResourceArgument()
+ configuration_attributes = ResourceArgument()
conf_dir = ResourceArgument()
mode = ResourceArgument()
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
index db9bb6c..a1ecbe5 100644
--- a/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestXmlConfigResource.py
@@ -61,8 +61,9 @@ class TestXmlConfigResource(TestCase):
with Environment('/') as env:
XmlConfig('file.xml',
conf_dir='/dir/conf',
- configurations={}
- )
+ configurations={},
+ configuration_attributes={}
+ )
open_mock.assert_called_with('/dir/conf/file.xml', 'wb')
result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n </configuration>\n')
@@ -93,11 +94,12 @@ class TestXmlConfigResource(TestCase):
with Environment('/') as env:
XmlConfig('file.xml',
conf_dir='/dir/conf',
- configurations={'property1': 'value1'}
- )
+ configurations={'property1': 'value1'},
+ configuration_attributes={'attr': {'property1': 'attr_value'}}
+ )
open_mock.assert_called_with('/dir/conf/file.xml', 'wb')
- result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name>property1</name>\n <value>value1</value>\n </property>\n \n </configuration>\n')
+ result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name>property1</name>\n <value>value1</value>\n <attr>attr_value</attr>\n </property>\n \n </configuration>\n')
@patch("resource_management.core.providers.system._ensure_metadata")
@@ -131,8 +133,27 @@ class TestXmlConfigResource(TestCase):
"prop.2": "INFO, openjpa",
"prop.4": "${oozie.log.dir}/oozie.log",
"prop.empty": "",
- },
- )
+ },
+ configuration_attributes={
+ "": {
+ "prop.1": "should_not_be_printed",
+ "prop.2": "should_not_be_printed",
+ },
+ "attr1": {
+ "prop.1": "x",
+ "prop.8": "not_existed",
+ },
+ "attr2": {
+ "prop.4": "value4",
+ "prop.3": "value3"
+ },
+ "attr_empty": {
+ },
+ "attr_value_empty": {
+ "prop.4": "",
+ "prop.empty": ""
+ }
+ })
open_mock.assert_called_with('/dir/conf/file.xml', 'wb')
- result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.empty</name>\n <value></value>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n </property>\n \n <property>\n <name>prop.4</name>\n <value>${oozie.log.dir}/oozie.log</value>\n </property>\n \n </configuration>\n')
+ result_file.__enter__().write.assert_called_with(u'<!--Wed 2014-02-->\n <configuration>\n \n <property>\n <name></name>\n <value></value>\n </property>\n \n <property>\n <name>prop.empty</name>\n <value></value>\n <attr_value_empty></attr_value_empty>\n </property>\n \n <property>\n <name>prop.3</name>\n <value>%d{ISO8601} %5p %c{1}:%L - %m%n</value>\n <attr2>value3</attr2>\n </property>\n \n <property>\n <name>prop.2</name>\n <value>INFO, openjpa</value>\n </property>\n \n <property>\n <name>prop.1</name>\n <value>'.'yyyy-MM-dd-HH</value>\n <attr1>x</attr1>\n </property>\n \n <property>\n <name>prop.4</name>\n <value>${oozie.log.dir}/oozie.log</value>\n <attr_value_empty></attr_value_empty>\n <attr2>value4</attr2>\n </property>\n \n </configuration>\n')
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 8bc14f4..79d8925 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -32,6 +32,7 @@ import org.apache.commons.logging.LogFactory;
import java.util.HashMap;
import java.util.Map;
+import java.util.TreeMap;
public class ExecutionCommandWrapper {
@Inject
@@ -96,6 +97,21 @@ public class ExecutionCommandWrapper {
}
}
+ Map<String, Map<String, Map<String, String>>> configAttributes = configHelper.getEffectiveConfigAttributes(cluster,
+ executionCommand.getConfigurationTags());
+
+ for (Map.Entry<String, Map<String, Map<String, String>>> attributesOccurance : configAttributes.entrySet()) {
+ String type = attributesOccurance.getKey();
+ Map<String, Map<String, String>> attributes = attributesOccurance.getValue();
+
+ if (executionCommand.getConfigurationAttributes() != null) {
+ if (!executionCommand.getConfigurationAttributes().containsKey(type)) {
+ executionCommand.getConfigurationAttributes().put(type, new TreeMap<String, Map<String, String>>());
+ }
+ configHelper.cloneAttributesMap(attributes, executionCommand.getConfigurationAttributes().get(type));
+ }
+ }
+
} catch (AmbariException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 882f1e3..07dd782 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -21,6 +21,7 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import com.google.gson.annotations.SerializedName;
import org.apache.ambari.server.RoleCommand;
import org.apache.ambari.server.utils.StageUtils;
import org.apache.commons.logging.Log;
@@ -51,6 +52,8 @@ public class ExecutionCommand extends AgentCommand {
private Map<String, Set<String>> clusterHostInfo =
new HashMap<String, Set<String>>();
private Map<String, Map<String, String>> configurations;
+ @SerializedName("configuration_attributes")
+ private Map<String, Map<String, Map<String, String>>> configurationAttributes;
private Map<String, Map<String, String>> configurationTags;
private Map<String, String> commandParams;
private String serviceName;
@@ -188,6 +191,16 @@ public class ExecutionCommand extends AgentCommand {
this.configurations = configurations;
}
+ @JsonProperty("configuration_attributes")
+ public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() {
+ return configurationAttributes;
+ }
+
+ @JsonProperty("configuration_attributes")
+ public void setConfigurationAttributes(Map<String, Map<String, Map<String, String>>> configurationAttributes) {
+ this.configurationAttributes = configurationAttributes;
+ }
+
@JsonProperty("commandParams")
public Map<String, String> getCommandParams() {
return commandParams;
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
index 8b8b252..d216aea 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartbeatMonitor.java
@@ -239,10 +239,10 @@ public class HeartbeatMonitor implements Runnable {
stackId.getStackVersion());
Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>();
// get the cluster config for type '*-env'
// apply config group overrides
-
//Config clusterConfig = cluster.getDesiredConfigByType(GLOBAL);
Collection<Config> clusterConfigs = cluster.getAllConfigs();
@@ -277,6 +277,16 @@ public class HeartbeatMonitor implements Runnable {
}
configurations.put(clusterConfig.getType(), props);
+
+ Map<String, Map<String, String>> attrs = new TreeMap<String, Map<String, String>>();
+ configHelper.cloneAttributesMap(clusterConfig.getPropertiesAttributes(), attrs);
+
+ Map<String, Map<String, Map<String, String>>> attributes = configHelper
+ .getEffectiveConfigAttributes(cluster, configTags);
+ for (Map<String, Map<String, String>> attributesMap : attributes.values()) {
+ configHelper.cloneAttributesMap(attributesMap, attrs);
+ }
+ configurationAttributes.put(clusterConfig.getType(), attrs);
}
}
@@ -295,6 +305,7 @@ public class HeartbeatMonitor implements Runnable {
statusCmd.setServiceName(serviceName);
statusCmd.setComponentName(componentName);
statusCmd.setConfigurations(configurations);
+ statusCmd.setConfigurationAttributes(configurationAttributes);
// Fill command params
Map<String, String> commandParams = statusCmd.getCommandParams();
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
index e7a46ff..a7da22b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/StatusCommand.java
@@ -21,6 +21,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.gson.annotations.SerializedName;
import org.codehaus.jackson.annotate.JsonProperty;
/**
@@ -36,6 +37,8 @@ public class StatusCommand extends AgentCommand {
private String serviceName;
private String componentName;
private Map<String, Map<String, String>> configurations;
+ @SerializedName("configuration_attributes")
+ private Map<String, Map<String, Map<String, String>>> configurationAttributes;
private Map<String, String> commandParams = new HashMap<String, String>();
private Map<String, String> hostLevelParams = new HashMap<String, String>();
@@ -79,6 +82,16 @@ public class StatusCommand extends AgentCommand {
this.configurations = configurations;
}
+ @JsonProperty("configuration_attributes")
+ public Map<String, Map<String, Map<String, String>>> getConfigurationAttributes() {
+ return configurationAttributes;
+ }
+
+ @JsonProperty("configuration_attributes")
+ public void setConfigurationAttributes(Map<String, Map<String, Map<String, String>>> configurationAttributes) {
+ this.configurationAttributes = configurationAttributes;
+ }
+
@JsonProperty("hostLevelParams")
public Map<String, String> getHostLevelParams() {
return hostLevelParams;
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 8ef40f6..64f132e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -331,6 +331,7 @@ public class AmbariActionExecutionHelper {
serviceName);
Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>();
Map<String, Map<String, String>> configTags = null;
if (!serviceName.isEmpty() && null != cluster) {
configTags = managementController.findConfigurationTagsWithOverrides(cluster, hostName);
@@ -350,6 +351,7 @@ public class AmbariActionExecutionHelper {
* combined with the same code at createHostAction()
*/
execCmd.setConfigurations(configurations);
+ execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
execCmd.setHostLevelParams(hostLevelParams);
execCmd.setCommandParams(commandParams);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index cb36f27..a05b9d4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -256,6 +256,8 @@ public class AmbariCustomCommandExecutionHelper {
Map<String, Map<String, String>> configurations =
new TreeMap<String, Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> configurationAttributes =
+ new TreeMap<String, Map<String, Map<String, String>>>();
Map<String, Map<String, String>> configTags =
managementController.findConfigurationTagsWithOverrides(cluster, hostName);
@@ -269,6 +271,7 @@ public class AmbariCustomCommandExecutionHelper {
componentName).getExecutionCommand();
execCmd.setConfigurations(configurations);
+ execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
@@ -447,6 +450,8 @@ public class AmbariCustomCommandExecutionHelper {
// [ type -> [ key, value ] ]
Map<String, Map<String, String>> configurations =
new TreeMap<String, Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> configurationAttributes =
+ new TreeMap<String, Map<String, Map<String, String>>>();
Map<String, Map<String, String>> configTags =
managementController.findConfigurationTagsWithOverrides(cluster, hostname);
@@ -454,6 +459,7 @@ public class AmbariCustomCommandExecutionHelper {
smokeTestRole).getExecutionCommand();
execCmd.setConfigurations(configurations);
+ execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
// Generate cluster host info
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 7d49c1b..dd99d87 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -1357,6 +1357,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
private void createHostAction(Cluster cluster,
Stage stage, ServiceComponentHost scHost,
Map<String, Map<String, String>> configurations,
+ Map<String, Map<String, Map<String, String>>> configurationAttributes,
Map<String, Map<String, String>> configTags,
RoleCommand roleCommand,
Map<String, String> commandParams,
@@ -1402,6 +1403,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
}
execCmd.setConfigurations(configurations);
+ execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
if (commandParams == null) { // if not defined
commandParams = new TreeMap<String, String>();
@@ -1741,6 +1743,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
// [ type -> [ key, value ] ]
Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, Map<String, Map<String, String>>> configurationAttributes = new TreeMap<String, Map<String, Map<String, String>>>();
Host host = clusters.getHost(scHost.getHostName());
Map<String, Map<String, String>> configTags =
@@ -1763,7 +1766,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
requestParameters.put(keyName, requestProperties.get(keyName));
}
- createHostAction(cluster, stage, scHost, configurations, configTags,
+ createHostAction(cluster, stage, scHost, configurations, configurationAttributes, configTags,
roleCommand, requestParameters, event);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 2f2f259..9575385 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -182,6 +182,55 @@ public class ConfigHelper {
}
/**
+ * Get all config attributes for a cluster given a set of configType to
+ * versionTags map. This helper method merges all the override tags with a
+ * the attributes from parent cluster config properties
+ *
+ * @param cluster
+ * @param desiredTags
+ * @return {type : {attribute : {property, attributeValue}}
+ */
+ public Map<String, Map<String, Map<String, String>>> getEffectiveConfigAttributes(
+ Cluster cluster, Map<String, Map<String, String>> desiredTags) {
+
+ Map<String, Map<String, Map<String, String>>> attributes = new HashMap<String, Map<String, Map<String, String>>>();
+
+ if (desiredTags != null) {
+ for (Entry<String, Map<String, String>> entry : desiredTags.entrySet()) {
+
+ String type = entry.getKey();
+ Map<String, Map<String, String>> attributesMap = null;
+
+ Map<String, String> tags = new HashMap<String, String>(entry.getValue());
+ String clusterTag = tags.get(CLUSTER_DEFAULT_TAG);
+
+ if (clusterTag != null) {
+ Config config = cluster.getConfig(type, clusterTag);
+ if (config != null) {
+ attributesMap = new TreeMap<String, Map<String, String>>();
+ cloneAttributesMap(config.getPropertiesAttributes(), attributesMap);
+ }
+ tags.remove(CLUSTER_DEFAULT_TAG);
+ }
+ for (Entry<String, String> overrideEntry : tags.entrySet()) {
+ Config overrideConfig = cluster.getConfig(type,
+ overrideEntry.getValue());
+
+ // TODO clarify correct behavior for attributes overriding
+ if (overrideConfig != null) {
+ cloneAttributesMap(overrideConfig.getPropertiesAttributes(), attributesMap);
+ }
+ }
+ if (attributesMap != null) {
+ attributes.put(type, attributesMap);
+ }
+ }
+ }
+
+ return attributes;
+ }
+
+ /**
* Merge override with original, if original property doesn't exist,
* add it to the properties
*
@@ -198,7 +247,7 @@ public class ConfigHelper {
for (Entry<String, String> entry : override.entrySet()) {
Boolean deleted = 0 == entry.getKey().indexOf(DELETED);
String nameToUse = deleted ?
- entry.getKey().substring(DELETED.length()) : entry.getKey();
+ entry.getKey().substring(DELETED.length()) : entry.getKey();
if (finalConfig.containsKey(nameToUse)) {
finalConfig.remove(nameToUse);
}
@@ -211,6 +260,21 @@ public class ConfigHelper {
return finalConfig;
}
+ public void cloneAttributesMap(Map<String, Map<String, String>> sourceAttributesMap,
+ Map<String, Map<String, String>> targetAttributesMap) {
+ if (sourceAttributesMap != null && targetAttributesMap != null) {
+ for (Entry<String, Map<String, String>> attributesEntry : sourceAttributesMap.entrySet()) {
+ String attributeName = attributesEntry.getKey();
+ if (!targetAttributesMap.containsKey(attributeName)) {
+ targetAttributesMap.put(attributeName, new TreeMap<String, String>());
+ }
+ for (Entry<String, String> attributesValue : attributesEntry.getValue().entrySet()) {
+ targetAttributesMap.get(attributeName).put(attributesValue.getKey(), attributesValue.getValue());
+ }
+ }
+ }
+ }
+
public void applyCustomConfig(Map<String, Map<String, String>> configurations,
String type, String name, String value, Boolean deleted) {
if (!configurations.containsKey(type)) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
index a7b4051..7160bcf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/utils/StageUtils.java
@@ -178,6 +178,14 @@ public class StageUtils {
new TreeMap<String, Map<String, String>>();
configurations.put("hdfs-site", hdfsSite);
execCmd.setConfigurations(configurations);
+ Map<String, Map<String, Map<String, String>>> configurationAttributes =
+ new TreeMap<String, Map<String, Map<String, String>>>();
+ Map<String, Map<String, String>> hdfsSiteAttributes = new TreeMap<String, Map<String, String>>();
+ Map<String, String> finalAttribute = new TreeMap<String, String>();
+ finalAttribute.put("dfs.block.size", "true");
+ hdfsSiteAttributes.put("final", finalAttribute);
+ configurationAttributes.put("hdfsSite", hdfsSiteAttributes);
+ execCmd.setConfigurationAttributes(configurationAttributes);
Map<String, String> params = new TreeMap<String, String>();
params.put("jdklocation", "/x/y/z");
params.put("stack_version", "1.2.0");
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/shared_initialization.py
index d37f1e3..9c8d413 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -46,6 +46,7 @@ def setup_config():
XmlConfig("core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
index 0dc5347..1734dda 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HBASE/package/scripts/hbase.py
@@ -41,6 +41,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig( "hbase-site.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hbase-site'],
+ configuration_attributes=params.config['configuration_attributes']['hbase-site'],
owner = params.hbase_user,
group = params.user_group
)
@@ -48,6 +49,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig( "hdfs-site.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner = params.hbase_user,
group = params.user_group
)
@@ -55,6 +57,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group
)
@@ -63,6 +66,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig( "hbase-policy.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hbase-policy'],
+ configuration_attributes=params.config['configuration_attributes']['hbase-policy'],
owner = params.hbase_user,
group = params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py
index d988c56..69009ca 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HDFS/package/scripts/hdfs.py
@@ -45,6 +45,7 @@ def hdfs(name=None):
XmlConfig("hadoop-policy.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hadoop-policy'],
+ configuration_attributes=params.config['configuration_attributes']['hadoop_policy'],
owner=params.hdfs_user,
group=params.user_group
)
@@ -52,6 +53,7 @@ def hdfs(name=None):
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hcat.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hcat.py
index c25f530..d14ea66 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hcat.py
@@ -38,6 +38,7 @@ def hcat():
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['hive-site'],
+ configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py
index 5ee6000..675dbe3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/HIVE/package/scripts/hive.py
@@ -56,6 +56,7 @@ def hive(name=None):
XmlConfig("hive-site.xml",
conf_dir=hive_config_dir,
configurations=params.config['configurations']['hive-site'],
+ configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=config_file_mode
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
index 9b2f387..2ab22b8 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/MAPREDUCE/package/scripts/mapreduce.py
@@ -127,6 +127,7 @@ def mapreduce(name=None):
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations'][
'capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group
)
@@ -136,6 +137,7 @@ def mapreduce(name=None):
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations'][
'mapred-queue-acls'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-queue-acls'],
owner=params.mapred_user,
group=params.user_group
)
@@ -150,6 +152,7 @@ def mapreduce(name=None):
XmlConfig("mapred-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.mapred_user,
group=params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py
index 5fd2c96..af06efc 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/OOZIE/package/scripts/oozie.py
@@ -34,6 +34,7 @@ def oozie(is_server=False
XmlConfig( "oozie-site.xml",
conf_dir = params.conf_dir,
configurations = params.config['configurations']['oozie-site'],
+ configuration_attributes=params.config['configuration_attributes']['oozie-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
index d97de6f..037cdb5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/1.3.2/services/WEBHCAT/package/scripts/webhcat.py
@@ -60,6 +60,7 @@ def webhcat():
XmlConfig("webhcat-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['webhcat-site'],
+ configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
owner=params.webhcat_user,
group=params.user_group,
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
index cde7a03..0b40ea6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -44,6 +44,7 @@ def setup_config():
XmlConfig("core-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group
)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
index 8fa4ceb..2829c7b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HBASE/package/scripts/hbase.py
@@ -47,6 +47,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig( "hbase-site.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hbase-site'],
+ configuration_attributes=params.config['configuration_attributes']['hbase-site'],
owner = params.hbase_user,
group = params.user_group
)
@@ -54,6 +55,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig( "hdfs-site.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner = params.hbase_user,
group = params.user_group
)
@@ -61,6 +63,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group
)
@@ -69,6 +72,7 @@ def hbase(name=None # 'master' or 'regionserver' or 'client'
XmlConfig( "hbase-policy.xml",
conf_dir = params.hbase_conf_dir,
configurations = params.config['configurations']['hbase-policy'],
+ configuration_attributes=params.config['configuration_attributes']['hbase-policy'],
owner = params.hbase_user,
group = params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
index d988c56..9985e32 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/hdfs.py
@@ -45,6 +45,7 @@ def hdfs(name=None):
XmlConfig("hadoop-policy.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hadoop-policy'],
+ configuration_attributes=params.config['configuration_attributes']['hadoop-policy'],
owner=params.hdfs_user,
group=params.user_group
)
@@ -52,6 +53,7 @@ def hdfs(name=None):
XmlConfig("hdfs-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
+ configuration_attributes=params.config['configuration_attributes']['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py
index c25f530..d14ea66 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hcat.py
@@ -38,6 +38,7 @@ def hcat():
XmlConfig("hive-site.xml",
conf_dir=params.hive_conf_dir,
configurations=params.config['configurations']['hive-site'],
+ configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py
index a4ca3e0..c48dfc1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/HIVE/package/scripts/hive.py
@@ -56,6 +56,7 @@ def hive(name=None):
XmlConfig("mapred-site.xml",
conf_dir=hive_config_dir,
configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.hive_user,
group=params.user_group,
mode=config_file_mode)
@@ -63,6 +64,7 @@ def hive(name=None):
XmlConfig("hive-site.xml",
conf_dir=hive_config_dir,
configurations=params.config['configurations']['hive-site'],
+ configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=config_file_mode)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py
index 5834088..6fa2787 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/OOZIE/package/scripts/oozie.py
@@ -37,6 +37,7 @@ def oozie(is_server=False # TODO: see if see can remove this
XmlConfig( "oozie-site.xml",
conf_dir = params.conf_dir,
configurations = oozie_site,
+ configuration_attributes=params.config['configuration_attributes']['oozie-site'],
owner = params.oozie_user,
group = params.user_group,
mode = 0664
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
index c4dc3d6..841a946 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/WEBHCAT/package/scripts/webhcat.py
@@ -62,6 +62,7 @@ def webhcat():
XmlConfig("webhcat-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['webhcat-site'],
+ configuration_attributes=params.config['configuration_attributes']['webhcat-site'],
owner=params.webhcat_user,
group=params.user_group,
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
index 6885086..7b71614 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/YARN/package/scripts/yarn.py
@@ -80,6 +80,7 @@ def yarn(name = None):
XmlConfig("core-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644
@@ -88,6 +89,7 @@ def yarn(name = None):
XmlConfig("mapred-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644
@@ -96,6 +98,7 @@ def yarn(name = None):
XmlConfig("yarn-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['yarn-site'],
+ configuration_attributes=params.config['configuration_attributes']['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644
@@ -104,6 +107,7 @@ def yarn(name = None):
XmlConfig("capacity-scheduler.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644
@@ -191,6 +195,7 @@ def yarn(name = None):
XmlConfig("mapred-site.xml",
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.mapred_user,
group=params.user_group
)
@@ -200,6 +205,7 @@ def yarn(name = None):
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations'][
'mapred-queue-acls'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-queue-acls'],
owner=params.mapred_user,
group=params.user_group
)
@@ -215,6 +221,7 @@ def yarn(name = None):
conf_dir=params.hadoop_conf_dir,
configurations=params.config['configurations'][
'capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/TEZ/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/TEZ/package/scripts/tez.py
index f3bef97..7ec78a3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/TEZ/package/scripts/tez.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/TEZ/package/scripts/tez.py
@@ -33,6 +33,7 @@ def tez():
XmlConfig( "tez-site.xml",
conf_dir = params.config_dir,
configurations = params.config['configurations']['tez-site'],
+ configuration_attributes=params.config['configuration_attributes']['tez-site'],
owner = params.tez_user,
group = params.user_group,
mode = 0664
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py
index eeedc46..48f32d2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/scripts/yarn.py
@@ -86,6 +86,7 @@ def yarn(name = None):
XmlConfig("core-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644
@@ -94,6 +95,7 @@ def yarn(name = None):
XmlConfig("mapred-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['mapred-site'],
+ configuration_attributes=params.config['configuration_attributes']['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644
@@ -102,6 +104,7 @@ def yarn(name = None):
XmlConfig("yarn-site.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['yarn-site'],
+ configuration_attributes=params.config['configuration_attributes']['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644
@@ -110,6 +113,7 @@ def yarn(name = None):
XmlConfig("capacity-scheduler.xml",
conf_dir=params.config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
+ configuration_attributes=params.config['configuration_attributes']['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py
index 5402506..62e3cd6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1/services/TEZ/package/scripts/tez.py
@@ -33,6 +33,7 @@ def tez():
XmlConfig( "tez-site.xml",
conf_dir = params.config_dir,
configurations = params.config['configurations']['tez-site'],
+ configuration_attributes=params.config['configuration_attributes']['tez-site'],
owner = params.tez_user,
group = params.user_group,
mode = 0664
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 5e8d007..e51cea3 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -4413,10 +4413,12 @@ public class AmbariManagementControllerTest {
Config config1 = cf.createNew(cluster, "global",
new HashMap<String, String>(){{ put("key1", "value1"); }}, new HashMap<String, Map<String,String>>());
config1.setVersionTag("version1");
+ config1.setPropertiesAttributes(new HashMap<String, Map<String, String>>(){{ put("attr1", new HashMap<String, String>()); }});
Config config2 = cf.createNew(cluster, "core-site",
new HashMap<String, String>(){{ put("key1", "value1"); }}, new HashMap<String, Map<String,String>>());
config2.setVersionTag("version1");
+ config2.setPropertiesAttributes(new HashMap<String, Map<String, String>>(){{ put("attr2", new HashMap<String, String>()); }});
cluster.addConfig(config1);
cluster.addConfig(config2);
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
index 4d30e3d..c0bc8e6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/ConfigHelperTest.java
@@ -86,6 +86,12 @@ public class ConfigHelperTest {
put("ipc.client.connect.max.retries", "30");
put("fs.trash.interval", "30");
}});
+ cr.setPropertiesAttributes(new HashMap<String, Map<String, String>>() {{
+ Map<String, String> attrs = new HashMap<String, String>();
+ attrs.put("ipc.client.connect.max.retries", "1");
+ attrs.put("fs.trash.interval", "2");
+ put("attribute1", attrs);
+ }});
final ClusterRequest clusterRequest1 =
new ClusterRequest(cluster.getClusterId(), clusterName,
@@ -102,6 +108,12 @@ public class ConfigHelperTest {
put("dfs_namenode_name_dir", "/hadoop/hdfs/namenode");
put("namenode_heapsize", "1024");
}});
+ cr.setPropertiesAttributes(new HashMap<String, Map<String, String>>() {{
+ Map<String, String> attrs = new HashMap<String, String>();
+ attrs.put("dfs_namenode_name_dir", "3");
+ attrs.put("namenode_heapsize", "4");
+ put("attribute2", attrs);
+ }});
final ClusterRequest clusterRequest2 =
new ClusterRequest(cluster.getClusterId(), clusterName,
@@ -206,4 +218,165 @@ public class ConfigHelperTest {
Assert.assertTrue(coreProps.containsKey("c"));
Assert.assertEquals("30", coreProps.get("ipc.client.connect.max.retries"));
}
+
+ @Test
+ public void testEffectivePropertiesAttributesWithOverrides() throws Exception {
+ final Config config1 = new ConfigImpl("core-site");
+ config1.setVersionTag("version122");
+
+ Map<String, String> attributes = new HashMap<String, String>();
+ attributes.put("fs.trash.interval", "11");
+ attributes.put("b", "y");
+ Map<String, Map<String, String>> config1Attributes = new HashMap<String, Map<String, String>>();
+ config1Attributes.put("attribute1", attributes);
+ config1.setPropertiesAttributes(config1Attributes);
+
+ final Config config2 = new ConfigImpl("global");
+ config2.setVersionTag("version122");
+ attributes = new HashMap<String, String>();
+ attributes.put("namenode_heapsize", "z");
+ attributes.put("c", "q");
+ Map<String, Map<String, String>> config2Attributes = new HashMap<String, Map<String, String>>();
+ config2Attributes.put("attribute2", attributes);
+ config2.setPropertiesAttributes(config2Attributes);
+
+ Long groupId = addConfigGroup("g1", "t1", new ArrayList<String>() {{
+ add("h1");}}, new ArrayList<Config>() {{ add(config1); add(config2);
+ }});
+
+ Assert.assertNotNull(groupId);
+
+ Map<String, Map<String, Map<String, String>>> effectiveAttributes = configHelper
+ .getEffectiveConfigAttributes(cluster,
+ configHelper.getEffectiveDesiredTags(cluster, "h1"));
+
+ Assert.assertNotNull(effectiveAttributes);
+ Assert.assertEquals(2, effectiveAttributes.size());
+
+ Assert.assertTrue(effectiveAttributes.containsKey("global"));
+ Map<String, Map<String, String>> globalAttrs = effectiveAttributes.get("global");
+ Assert.assertEquals(1, globalAttrs.size());
+ Assert.assertTrue(globalAttrs.containsKey("attribute2"));
+ Map<String, String> attribute2Occurances = globalAttrs.get("attribute2");
+ Assert.assertEquals(3, attribute2Occurances.size());
+ Assert.assertTrue(attribute2Occurances.containsKey("namenode_heapsize"));
+ Assert.assertEquals("z", attribute2Occurances.get("namenode_heapsize"));
+ Assert.assertTrue(attribute2Occurances.containsKey("dfs_namenode_name_dir"));
+ Assert.assertEquals("3", attribute2Occurances.get("dfs_namenode_name_dir"));
+ Assert.assertTrue(attribute2Occurances.containsKey("c"));
+ Assert.assertEquals("q", attribute2Occurances.get("c"));
+
+ Assert.assertTrue(effectiveAttributes.containsKey("core-site"));
+ Map<String, Map<String, String>> coreAttrs = effectiveAttributes.get("core-site");
+ Assert.assertEquals(1, coreAttrs.size());
+ Assert.assertTrue(coreAttrs.containsKey("attribute1"));
+ Map<String, String> attribute1Occurances = coreAttrs.get("attribute1");
+ Assert.assertEquals(3, attribute1Occurances.size());
+ Assert.assertTrue(attribute1Occurances.containsKey("ipc.client.connect.max.retries"));
+ Assert.assertEquals("1", attribute1Occurances.get("ipc.client.connect.max.retries"));
+ Assert.assertTrue(attribute1Occurances.containsKey("fs.trash.interval"));
+ Assert.assertEquals("11", attribute1Occurances.get("fs.trash.interval"));
+ Assert.assertTrue(attribute1Occurances.containsKey("b"));
+ Assert.assertEquals("y", attribute1Occurances.get("b"));
+ }
+
+ @Test
+ public void testCloneAttributesMap() throws Exception {
+ // init
+ Map<String, Map<String, String>> targetAttributesMap = new HashMap<String, Map<String, String>>();
+ Map<String, String> attributesValues = new HashMap<String, String>();
+ attributesValues.put("a", "1");
+ attributesValues.put("b", "2");
+ attributesValues.put("f", "3");
+ attributesValues.put("q", "4");
+ targetAttributesMap.put("attr", attributesValues);
+ Map<String, Map<String, String>> sourceAttributesMap = new HashMap<String, Map<String, String>>();
+ attributesValues = new HashMap<String, String>();
+ attributesValues.put("a", "5");
+ attributesValues.put("f", "6");
+ sourceAttributesMap.put("attr", attributesValues);
+ attributesValues = new HashMap<String, String>();
+ attributesValues.put("f", "7");
+ attributesValues.put("q", "8");
+ sourceAttributesMap.put("attr1", attributesValues);
+
+ // eval
+ configHelper.cloneAttributesMap(sourceAttributesMap, targetAttributesMap);
+
+ // verification
+ Assert.assertEquals(2, targetAttributesMap.size());
+ Assert.assertTrue(targetAttributesMap.containsKey("attr"));
+ Assert.assertTrue(targetAttributesMap.containsKey("attr1"));
+ Map<String, String> attributes = targetAttributesMap.get("attr");
+ Assert.assertEquals(4, attributes.size());
+ Assert.assertEquals("5", attributes.get("a"));
+ Assert.assertEquals("2", attributes.get("b"));
+ Assert.assertEquals("6", attributes.get("f"));
+ Assert.assertEquals("4", attributes.get("q"));
+ attributes = targetAttributesMap.get("attr1");
+ Assert.assertEquals(2, attributes.size());
+ Assert.assertEquals("7", attributes.get("f"));
+ Assert.assertEquals("8", attributes.get("q"));
+ }
+
+ @Test
+ public void testCloneAttributesMap_sourceIsNull() throws Exception {
+ // init
+ Map<String, Map<String, String>> targetAttributesMap = new HashMap<String, Map<String, String>>();
+ Map<String, String> attributesValues = new HashMap<String, String>();
+ attributesValues.put("a", "1");
+ attributesValues.put("b", "2");
+ attributesValues.put("f", "3");
+ attributesValues.put("q", "4");
+ targetAttributesMap.put("attr", attributesValues);
+ Map<String, Map<String, String>> sourceAttributesMap = null;
+
+ // eval
+ configHelper.cloneAttributesMap(sourceAttributesMap, targetAttributesMap);
+
+ // verification
+ // No exception should be thrown
+ // targetMap should not be changed
+ Assert.assertEquals(1, targetAttributesMap.size());
+ Assert.assertTrue(targetAttributesMap.containsKey("attr"));
+ Map<String, String> attributes = targetAttributesMap.get("attr");
+ Assert.assertEquals(4, attributes.size());
+ Assert.assertEquals("1", attributes.get("a"));
+ Assert.assertEquals("2", attributes.get("b"));
+ Assert.assertEquals("3", attributes.get("f"));
+ Assert.assertEquals("4", attributes.get("q"));
+ }
+
+ @Test
+ public void testCloneAttributesMap_targetIsNull() throws Exception {
+ // init
+ Map<String, Map<String, String>> targetAttributesMap = null;
+ Map<String, Map<String, String>> sourceAttributesMap = new HashMap<String, Map<String, String>>();
+ Map<String, String> attributesValues = new HashMap<String, String>();
+ attributesValues.put("a", "5");
+ attributesValues.put("f", "6");
+ sourceAttributesMap.put("attr", attributesValues);
+ attributesValues = new HashMap<String, String>();
+ attributesValues.put("f", "7");
+ attributesValues.put("q", "8");
+ sourceAttributesMap.put("attr1", attributesValues);
+
+ // eval
+ configHelper.cloneAttributesMap(sourceAttributesMap, targetAttributesMap);
+
+ // verification
+ // No exception should be thrown
+ // sourceMap should not be changed
+ Assert.assertEquals(2, sourceAttributesMap.size());
+ Assert.assertTrue(sourceAttributesMap.containsKey("attr"));
+ Assert.assertTrue(sourceAttributesMap.containsKey("attr1"));
+ Map<String, String> attributes = sourceAttributesMap.get("attr");
+ Assert.assertEquals(2, attributes.size());
+ Assert.assertEquals("5", attributes.get("a"));
+ Assert.assertEquals("6", attributes.get("f"));
+ attributes = sourceAttributesMap.get("attr1");
+ Assert.assertEquals(2, attributes.size());
+ Assert.assertEquals("7", attributes.get("f"));
+ Assert.assertEquals("8", attributes.get("q"));
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/TestResourceFilesKeeper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestResourceFilesKeeper.py b/ambari-server/src/test/python/TestResourceFilesKeeper.py
index a62cbb9..84b206d 100644
--- a/ambari-server/src/test/python/TestResourceFilesKeeper.py
+++ b/ambari-server/src/test/python/TestResourceFilesKeeper.py
@@ -49,7 +49,7 @@ class TestResourceFilesKeeper(TestCase):
DUMMY_UNCHANGEABLE_PACKAGE=os.path.join(DUMMY_UNCHANGEABLE_STACK,
ResourceFilesKeeper.PACKAGE_DIR)
- DUMMY_UNCHANGEABLE_PACKAGE_HASH="4cd211c4d16b49b3546ecc9b7996698105fe10e4"
+ DUMMY_UNCHANGEABLE_PACKAGE_HASH="5de50cd1b26c5ff9b6f1031b9d15fa14df1a3898"
DUMMY_HASH="dummy_hash"
YA_HASH="yet_another_hash"
SOME_PATH="some-path"
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
index 91efd0d..479b407 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_client.py
@@ -43,19 +43,22 @@ class TestHBaseClient(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
@@ -106,19 +109,22 @@ class TestHBaseClient(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_master.py
index 9dca806..1ae7eb8 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_master.py
@@ -177,19 +177,22 @@ class TestHBaseMaster(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
@@ -264,19 +267,22 @@ class TestHBaseMaster(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_regionserver.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_regionserver.py b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_regionserver.py
index a899dec..88570d5 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_regionserver.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HBASE/test_hbase_regionserver.py
@@ -111,19 +111,22 @@ class TestHbaseRegionServer(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
@@ -199,19 +202,22 @@ class TestHbaseRegionServer(RMFTestCase):
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hbase-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hbase-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hbase',
group = 'hadoop',
conf_dir = '/etc/hbase/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
owner = 'hdfs',
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
- configurations = self.getConfig()['configurations']['hdfs-site'], # don't hardcode all the properties
+ configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hbase/conf/hbase-policy.xml',
owner = 'hbase',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_datanode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_datanode.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_datanode.py
index e796ca6..7305779 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_datanode.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_datanode.py
@@ -156,6 +156,7 @@ class TestDatanode(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
content = Template('slaves.j2'),
@@ -192,6 +193,7 @@ class TestDatanode(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
content = Template('slaves.j2'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_namenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_namenode.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_namenode.py
index 91beb6e..223b401 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_namenode.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_namenode.py
@@ -229,6 +229,7 @@ class TestNamenode(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
@@ -256,6 +257,7 @@ class TestNamenode(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_snamenode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_snamenode.py b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_snamenode.py
index 951013c..8f053adb 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_snamenode.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HDFS/test_snamenode.py
@@ -177,6 +177,7 @@ class TestSNamenode(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
content = Template('slaves.j2'),
@@ -201,6 +202,7 @@ class TestSNamenode(RMFTestCase):
group = 'hadoop',
conf_dir = '/etc/hadoop/conf',
configurations = self.getConfig()['configurations']['hdfs-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site']
)
self.assertResourceCalled('File', '/etc/hadoop/conf/slaves',
content = Template('slaves.j2'),
http://git-wip-us.apache.org/repos/asf/ambari/blob/a1e9fa9a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hcat_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hcat_client.py b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hcat_client.py
index a0f0dab..a57455b 100644
--- a/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hcat_client.py
+++ b/ambari-server/src/test/python/stacks/1.3.2/HIVE/test_hcat_client.py
@@ -43,6 +43,7 @@ class TestHcatClient(RMFTestCase):
mode = 0644,
conf_dir = '/etc/hive/conf',
configurations = self.getConfig()['configurations']['hive-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hive-site']
)
self.assertResourceCalled('TemplateConfig', '/etc/hcatalog/conf/hcat-env.sh',
@@ -75,6 +76,7 @@ class TestHcatClient(RMFTestCase):
mode = 0644,
conf_dir = '/etc/hive/conf',
configurations = self.getConfig()['configurations']['hive-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['hive-site']
)
self.assertResourceCalled('TemplateConfig', '/etc/hcatalog/conf/hcat-env.sh',