You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2013/09/17 04:53:47 UTC
git commit: AMBARI-3142. Add support for task specific custom
configuration.
Updated Branches:
refs/heads/trunk 72438ae29 -> 032055dcb
AMBARI-3142. Add support for task specific custom configuration.
Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/032055dc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/032055dc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/032055dc
Branch: refs/heads/trunk
Commit: 032055dcb908358e82963a29856e89533551936b
Parents: 72438ae
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Sep 16 19:52:09 2013 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Sep 16 19:52:09 2013 -0700
----------------------------------------------------------------------
.../actionmanager/ExecutionCommandWrapper.java | 91 +++++++++++++-------
.../server/configuration/Configuration.java | 2 +
.../AmbariManagementControllerImpl.java | 25 +++---
.../AmbariManagementControllerTest.java | 46 +++++++++-
4 files changed, 115 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
index 182b7c6..cbce788 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/actionmanager/ExecutionCommandWrapper.java
@@ -33,10 +33,10 @@ import java.util.HashMap;
import java.util.Map;
public class ExecutionCommandWrapper {
- private static Log LOG = LogFactory.getLog(ExecutionCommandWrapper.class);
@Inject
static Injector injector;
-
+ private static Log LOG = LogFactory.getLog(ExecutionCommandWrapper.class);
+ private static String DELETED = "DELETED_";
String jsonExecutionCommand = null;
ExecutionCommand executionCommand = null;
@@ -48,59 +48,90 @@ public class ExecutionCommandWrapper {
this.executionCommand = executionCommand;
}
+ public static void applyCustomConfig(Map<String, Map<String, String>> configurations, String type,
+ String name, String value, Boolean deleted) {
+ if (!configurations.containsKey(type)) {
+ configurations.put(type, new HashMap<String, String>());
+ }
+ String nameToUse = deleted ? DELETED + name : name;
+ Map<String, String> properties = configurations.get(type);
+ if (properties.containsKey(nameToUse)) {
+ properties.remove(nameToUse);
+ }
+ properties.put(nameToUse, value);
+ }
+
+ public static Map<String, String> getMergedConfig(Map<String, String> persistedClusterConfig,
+ Map<String, String> override) {
+ Map<String, String> finalConfig = new HashMap<String, String>(persistedClusterConfig);
+ if (override != null && override.size() > 0) {
+ for (String overrideKey : override.keySet()) {
+ Boolean deleted = 0 == overrideKey.indexOf(DELETED);
+ String nameToUse = deleted ? overrideKey.substring(DELETED.length()) : overrideKey;
+ if (finalConfig.containsKey(nameToUse)) {
+ finalConfig.remove(nameToUse);
+ }
+ if (!deleted) {
+ finalConfig.put(nameToUse, override.get(overrideKey));
+ }
+ }
+ }
+
+ return finalConfig;
+ }
+
public ExecutionCommand getExecutionCommand() {
if (executionCommand != null) {
return executionCommand;
} else if (jsonExecutionCommand != null) {
-// try {
executionCommand = StageUtils.getGson().fromJson(jsonExecutionCommand, ExecutionCommand.class);
if (injector == null) {
throw new RuntimeException("Injector not found, configuration cannot be restored");
- } else if ((executionCommand.getConfigurations() == null || executionCommand.getConfigurations().isEmpty()) &&
- executionCommand.getConfigurationTags() != null &&
+ } else if (executionCommand.getConfigurationTags() != null &&
!executionCommand.getConfigurationTags().isEmpty()) {
+ // For a configuration type, both tag and an actual configuration can be stored
+ // Configurations from the tag is always expanded and then over-written by the actual
+ // global:version1:{a1:A1,b1:B1,d1:D1} + global:{a1:A2,c1:C1,DELETED_d1:x} ==>
+ // global:{a1:A2,b1:B1,c1:C1}
Clusters clusters = injector.getInstance(Clusters.class);
HostRoleCommandDAO hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class);
- Long clusterId = hostRoleCommandDAO.findByPK(executionCommand.getTaskId()).getStage().getCluster().getClusterId();
+ Long clusterId = hostRoleCommandDAO.findByPK(
+ executionCommand.getTaskId()).getStage().getCluster().getClusterId();
try {
Cluster cluster = clusters.getClusterById(clusterId);
- Map<String, Map<String, String>> configurations = new HashMap<String, Map<String, String>>();
-
for (Map.Entry<String, Map<String, String>> entry : executionCommand.getConfigurationTags().entrySet()) {
String type = entry.getKey();
Map<String, String> tags = entry.getValue();
- if (!configurations.containsKey(type)) {
- configurations.put(type, new HashMap<String, String>());
- }
-
- String tag;
-
- //perform override
//TODO align with configs override logic
- tag = tags.get("host_override_tag");
+ String tag = tags.get("host_override_tag");
tag = tag == null ? tags.get("service_override_tag") : tag;
tag = tag == null ? tags.get("tag") : tag;
if (tag != null) {
Config config = cluster.getConfig(type, tag);
- configurations.get(type).putAll(config.getProperties());
+ if (executionCommand.getConfigurations().containsKey(type)) {
+ Map<String, String> mergedConfig =
+ getMergedConfig(config.getProperties(), executionCommand.getConfigurations().get(type));
+ executionCommand.getConfigurations().get(type).clear();
+ executionCommand.getConfigurations().get(type).putAll(mergedConfig);
+
+ } else {
+ executionCommand.getConfigurations().put(type, new HashMap<String, String>());
+ executionCommand.getConfigurations().get(type).putAll(config.getProperties());
+ }
}
}
- executionCommand.setConfigurations(configurations);
} catch (AmbariException e) {
throw new RuntimeException(e);
}
}
return executionCommand;
-// } catch (IOException e) {
-// throw new RuntimeException(e);
-// }
} else {
throw new RuntimeException(
"Invalid ExecutionCommandWrapper, both object and string"
@@ -112,14 +143,8 @@ public class ExecutionCommandWrapper {
if (jsonExecutionCommand != null) {
return jsonExecutionCommand;
} else if (executionCommand != null) {
-// try {
- jsonExecutionCommand = StageUtils.getGson().toJson(executionCommand);
- return jsonExecutionCommand;
-// } catch (JAXBException e) {
-// throw new RuntimeException(e);
-// } catch (IOException e) {
-// throw new RuntimeException(e);
-// }
+ jsonExecutionCommand = StageUtils.getGson().toJson(executionCommand);
+ return jsonExecutionCommand;
} else {
throw new RuntimeException(
"Invalid ExecutionCommandWrapper, both object and string"
@@ -129,13 +154,15 @@ public class ExecutionCommandWrapper {
@Override
public boolean equals(Object o) {
- if (this == o)
+ if (this == o) {
return true;
- if (o == null || getClass() != o.getClass())
+ }
+ if (o == null || getClass() != o.getClass()) {
return false;
+ }
ExecutionCommandWrapper wrapper = (ExecutionCommandWrapper) o;
-
+
if (executionCommand != null && wrapper.executionCommand != null) {
return executionCommand.equals(wrapper.executionCommand);
} else {
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
index 04828f0..4364a79 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/configuration/Configuration.java
@@ -235,6 +235,8 @@ public class Configuration {
private static final String SERVER_CONNECTION_MAX_IDLE_TIME =
"server.connection.max.idle.millis";
+ public static final String GLOBAL_CONFIG_TAG = "global";
+ public static final String RCA_ENABLED_PROPERTY = "rca_enabled";
public static final String HIVE_CONFIG_TAG = "hive-site";
public static final String HIVE_METASTORE_PASSWORD_PROPERTY =
"javax.jdo.option.ConnectionPassword";
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index a280907..dd37821 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -45,6 +45,7 @@ import org.apache.ambari.server.ServiceComponentNotFoundException;
import org.apache.ambari.server.ServiceNotFoundException;
import org.apache.ambari.server.StackAccessException;
import org.apache.ambari.server.actionmanager.ActionManager;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.actionmanager.HostRoleStatus;
import org.apache.ambari.server.actionmanager.RequestStatus;
@@ -976,11 +977,8 @@ public class AmbariManagementControllerImpl implements
// Hack - Remove passwords from configs
if (event.getServiceComponentName().equals(Role.HIVE_CLIENT.toString())) {
- Map<String, String> hiveConfigs = configurations.get(Configuration
- .HIVE_CONFIG_TAG);
- if (hiveConfigs != null) {
- hiveConfigs.remove(Configuration.HIVE_METASTORE_PASSWORD_PROPERTY);
- }
+ ExecutionCommandWrapper.applyCustomConfig(configurations, Configuration.HIVE_CONFIG_TAG,
+ Configuration.HIVE_METASTORE_PASSWORD_PROPERTY, "", true);
}
execCmd.setConfigurations(configurations);
@@ -2053,7 +2051,7 @@ public class AmbariManagementControllerImpl implements
RoleCommand roleCommand;
State oldSchState = scHost.getState();
ServiceComponentHostEvent event;
- switch(newState) {
+ switch (newState) {
case INSTALLED:
if (oldSchState == State.INIT
|| oldSchState == State.UNINSTALLED
@@ -2162,20 +2160,19 @@ public class AmbariManagementControllerImpl implements
}
// [ type -> [ key, value ] ]
- Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String,String>>();
- Map<String, Map<String, String>> configTags = new HashMap<String, Map<String,String>>();
+ Map<String, Map<String, String>> configurations = new TreeMap<String, Map<String, String>>();
+ Map<String, Map<String, String>> configTags = new HashMap<String, Map<String, String>>();
findConfigurationPropertiesWithOverrides(configurations, configTags, cluster, scHost.getServiceName(),
scHost.getHostName(), clusterDesiredConfigs, configsByHosts.get(scHost.getHostName()));
// HACK HACK HACK
- if ((!scHost.getHostName().equals(jobtrackerHost))
- && configurations.get("global") != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting rca_enabled to false for host "
- + scHost.getHostName());
+ if (!scHost.getHostName().equals(jobtrackerHost)) {
+ if (configTags.get(Configuration.GLOBAL_CONFIG_TAG) != null) {
+ ExecutionCommandWrapper.applyCustomConfig(
+ configurations, Configuration.GLOBAL_CONFIG_TAG,
+ Configuration.RCA_ENABLED_PROPERTY, "false", false);
}
- configurations.get("global").put("rca_enabled", "false");
}
createHostAction(cluster, stage, scHost, configurations, configTags,
http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/032055dc/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 6336bfe..d2bf1e6 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -5333,13 +5333,15 @@ public class AmbariManagementControllerTest {
// Install
installService(clusterName, serviceName, false, false);
// Start
- long requestId = startService(clusterName, serviceName, false, false);
+ long requestId = startService(clusterName, serviceName, false, true);
String passwordInConfig = null;
+ Boolean isClientInstalled = false;
List<Stage> stages = actionDB.getAllStages(requestId);
for (Stage s : stages) {
for (HostRoleCommand hrc : s.getOrderedHostRoleCommands()) {
- if (hrc.getRole().equals(Role.HIVE_CLIENT.toString())) {
+ if (hrc.getRole().equals(Role.HIVE_CLIENT)) {
+ isClientInstalled = true;
Map<String, String> hiveSite = hrc.getExecutionCommandWrapper()
.getExecutionCommand().getConfigurations().get("hive-site");
Assert.assertNotNull(hiveSite);
@@ -5349,6 +5351,7 @@ public class AmbariManagementControllerTest {
}
}
}
+ Assert.assertTrue("HIVE_CLIENT must be installed", isClientInstalled);
Assert.assertNull(passwordInConfig);
}
@@ -7219,7 +7222,44 @@ public class AmbariManagementControllerTest {
controller.deleteHostComponents(schRequests);
Assert.assertEquals(0, cluster.getServiceComponentHosts(host1).size());
- }
+ }
+
+ @Test
+ public void testExecutionCommandConfiguration() throws AmbariException {
+ Map<String, Map<String, String>> config = new HashMap<String, Map<String, String>>();
+ config.put("type1", new HashMap<String, String>());
+ config.put("type3", new HashMap<String, String>());
+ config.get("type3").put("name1", "neverchange");
+ ExecutionCommandWrapper.applyCustomConfig(config, "type1", "name1", "value11", false);
+ Assert.assertEquals("value11", config.get("type1").get("name1"));
+
+ config.put("type1", new HashMap<String, String>());
+ ExecutionCommandWrapper.applyCustomConfig(config, "type1", "name1", "value12", false);
+ Assert.assertEquals("value12", config.get("type1").get("name1"));
+
+ ExecutionCommandWrapper.applyCustomConfig(config, "type2", "name2", "value21", false);
+ Assert.assertEquals("value21", config.get("type2").get("name2"));
+
+ ExecutionCommandWrapper.applyCustomConfig(config, "type2", "name2", "", true);
+ Assert.assertEquals("", config.get("type2").get("DELETED_name2"));
+ Assert.assertEquals("neverchange", config.get("type3").get("name1"));
+
+ Map<String, String> persistedClusterConfig = new HashMap<String, String>();
+ persistedClusterConfig.put("name1", "value11");
+ persistedClusterConfig.put("name3", "value31");
+ persistedClusterConfig.put("name4", "value41");
+ Map<String, String> override = new HashMap<String, String>();
+ override.put("name1", "value12");
+ override.put("name2", "value21");
+ override.put("DELETED_name3", "value31");
+ Map<String, String> mergedConfig = ExecutionCommandWrapper.getMergedConfig(persistedClusterConfig,
+ override);
+ Assert.assertEquals(3, mergedConfig.size());
+ Assert.assertFalse(mergedConfig.containsKey("name3"));
+ Assert.assertEquals("value12", mergedConfig.get("name1"));
+ Assert.assertEquals("value21", mergedConfig.get("name2"));
+ Assert.assertEquals("value41", mergedConfig.get("name4"));
+ }
}
\ No newline at end of file