You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2016/12/09 16:01:59 UTC
[1/2] ambari git commit: Revert "Revert "AMBARI-18888: Ambari-agent:
Create configuration files with JCEKS information""
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 2a8408118 -> d22422ba1
Revert "Revert "AMBARI-18888: Ambari-agent: Create configuration files with JCEKS information""
Set the value of credential store enabled on the execution command in AmbariManagementController::createHostAction() where the cluster and service are available.
This reverts commit 60a6bd4575fb87fc26c4a277cbabf850ef2089e1.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/24ac5cda
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/24ac5cda
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/24ac5cda
Branch: refs/heads/branch-2.5
Commit: 24ac5cdae228c0f421034f5c897f0635eb9bf52e
Parents: 2a84081
Author: Nahappan Somasundaram <ns...@hortonworks.com>
Authored: Fri Dec 2 08:23:43 2016 -0800
Committer: Nahappan Somasundaram <ns...@hortonworks.com>
Committed: Fri Dec 9 07:56:06 2016 -0800
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 3 +
.../ambari_agent/CustomServiceOrchestrator.py | 120 +++++++++++++++++++
ambari-agent/src/packages/tarball/all.xml | 30 +++++
.../ambari/server/agent/ExecutionCommand.java | 28 +++++
.../ambari/server/agent/HeartBeatHandler.java | 2 +-
.../AmbariManagementControllerImpl.java | 5 +
.../server/agent/TestHeartbeatHandler.java | 28 +++--
7 files changed, 207 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index c1d4c02..f2c8846 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -46,6 +46,9 @@ keysdir=/var/lib/ambari-agent/keys
server_crt=ca.crt
passphrase_env_var_name=AMBARI_PASSPHRASE
ssl_verify_cert=0
+credential_lib_dir=/var/lib/ambari-agent/cred/lib
+credential_conf_dir=/var/lib/ambari-agent/cred/conf
+credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell
[services]
pidLookupPath=/var/run/
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 11c8cbe..f9ed4cf 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,6 +29,8 @@ from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
from PythonReflectiveExecutor import PythonReflectiveExecutor
+from resource_management.core.utils import PasswordString
+import subprocess
import Constants
import hostname
@@ -65,6 +67,11 @@ class CustomServiceOrchestrator():
REFLECTIVELY_RUN_COMMANDS = FREQUENT_COMMANDS # -- commands which run a lot and often (this increases their speed)
DONT_BACKUP_LOGS_FOR_COMMANDS = FREQUENT_COMMANDS
+ # Path where hadoop credential JARS will be available
+ DEFAULT_CREDENTIAL_SHELL_LIB_PATH = '/var/lib/ambari-agent/cred/lib'
+ DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
+ DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell'
+
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
@@ -78,6 +85,14 @@ class CustomServiceOrchestrator():
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
+ # Construct the hadoop credential lib JARs path
+ self.credential_shell_lib_path = os.path.join(config.get('security', 'credential_lib_dir',
+ self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*')
+
+ self.credential_conf_dir = config.get('security', 'credential_conf_dir', self.DEFAULT_CREDENTIAL_CONF_DIR)
+
+ self.credential_shell_cmd = config.get('security', 'credential_shell_cmd', self.DEFAULT_CREDENTIAL_SHELL_CMD)
+
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -114,6 +129,102 @@ class CustomServiceOrchestrator():
else:
return PythonExecutor(self.tmp_dir, self.config)
+ def getProviderDirectory(self, service_name):
+ """
+ Gets the path to the service conf folder where the JCEKS file will be created.
+
+ :param service_name: Name of the service, for example, HIVE
+ :return: lower case path to the service conf folder
+ """
+
+ # The stack definition scripts of the service can move the
+ # JCEKS file around to where it wants, which is usually
+ # /etc/<service_name>/conf
+
+ conf_dir = os.path.join(self.credential_conf_dir, service_name.lower())
+ if not os.path.exists(conf_dir):
+ os.makedirs(conf_dir, 0644)
+
+ return conf_dir
+
+ def getAffectedConfigTypes(self, commandJson):
+ """
+ Gets the affected config types for the service in this command
+
+ :param commandJson:
+ :return:
+ """
+ return commandJson.get('configuration_attributes')
+
+ def getCredentialProviderPropertyName(self):
+ """
+ Gets the property name used by the hadoop credential provider
+ :return:
+ """
+ return 'hadoop.security.credential.provider.path'
+
+ def generateJceks(self, commandJson):
+ """
+ Generates the JCEKS file with passwords for the service specified in commandJson
+
+ :param commandJson: command JSON
+ :return: An exit value from the external process that generated the JCEKS file. None if
+ there are no passwords in the JSON.
+ """
+ cmd_result = None
+ roleCommand = None
+ if 'roleCommand' in commandJson:
+ roleCommand = commandJson['roleCommand']
+
+ logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
+
+ # Password properties for a config type, if present,
+ # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2}
+ passwordProperties = {}
+ config_types = self.getAffectedConfigTypes(commandJson)
+ for config_type in config_types:
+ elem = config_types.get(config_type)
+ hidden = elem.get('hidden')
+ if hidden is not None:
+ passwordProperties[config_type] = hidden
+
+ # Set up the variables for the external command to generate a JCEKS file
+ java_home = commandJson['hostLevelParams']['java_home']
+ java_bin = '{java_home}/bin/java'.format(java_home=java_home)
+
+ cs_lib_path = self.credential_shell_lib_path
+ serviceName = commandJson['serviceName']
+
+ # Gather the password values and remove them from the configuration
+ configs = commandJson.get('configurations')
+ for key, value in passwordProperties.items():
+ config = configs.get(key)
+ if config is not None:
+ file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key))
+ if os.path.exists(file_path):
+ os.remove(file_path)
+ provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
+ logger.info('provider_path={0}'.format(provider_path))
+ for alias in value:
+ pwd = config.get(alias)
+ if pwd is not None:
+ # Remove the clear text password
+ config.pop(alias, None)
+ # Add JCEKS provider path instead
+ config[self.getCredentialProviderPropertyName()] = provider_path
+ logger.debug("config={0}".format(config))
+ protected_pwd = PasswordString(pwd)
+ # Generate the JCEKS file
+ cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
+ alias, '-value', protected_pwd, '-provider', provider_path)
+ logger.info(cmd)
+ cmd_result = subprocess.call(cmd)
+ logger.info('cmd_result = {0}'.format(cmd_result))
+ os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
+
+ return cmd_result
+
+
def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name=None,
override_output_files=True, retry=False):
"""
@@ -174,6 +285,15 @@ class CustomServiceOrchestrator():
handle.on_background_command_started = self.map_task_to_process
del command['__handle']
+ # If command contains credentialStoreEnabled, then
+ # generate the JCEKS file for the configurations.
+ credentialStoreEnabled = False
+ if 'credentialStoreEnabled' in command:
+ credentialStoreEnabled = (command['credentialStoreEnabled'] == "true")
+
+ if credentialStoreEnabled == True:
+ self.generateJceks(command)
+
json_path = self.dump_command_to_json(command, retry)
pre_hook_tuple = self.resolve_hook_script_path(hook_dir,
self.PRE_HOOK_PREFIX, command_name, script_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-agent/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/src/packages/tarball/all.xml b/ambari-agent/src/packages/tarball/all.xml
index c481208..f8a54e3 100644
--- a/ambari-agent/src/packages/tarball/all.xml
+++ b/ambari-agent/src/packages/tarball/all.xml
@@ -190,4 +190,34 @@
<outputDirectory>/var/lib/${project.artifactId}/data</outputDirectory>
</file>
</files>
+ <moduleSets>
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
+ <unpack>false</unpack>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
+ <unpack>false</unpack>
+ <includes>
+ <include>commons-cli:commons-cli</include>
+ <include>commons-collections:commons-collections</include>
+ <include>commons-configuration:commons-configuration</include>
+ <include>commons-io:commons-io:jar:${commons.io.version}</include>
+ <include>commons-lang:commons-lang</include>
+ <include>commons-logging:commons-logging</include>
+ <include>com.google.guava:guava</include>
+ <include>org.slf4j:slf4j-api</include>
+ <include>org.apache.hadoop:hadoop-common</include>
+ <include>org.apache.hadoop:hadoop-auth</include>
+ <include>org.apache.htrace:htrace-core</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+ </binaries>
+ </moduleSet>
+ </moduleSets>
</assembly>
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index ef1ee4f..e46167a 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -112,6 +112,13 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("availableServices")
private Map<String, String> availableServices = new HashMap<>();
+ /**
+ * "true" or "false" indicating whether this
+ * service is enabled for credential store use.
+ */
+ @SerializedName("credentialStoreEnabled")
+ private String credentialStoreEnabled;
+
public String getCommandId() {
return commandId;
}
@@ -295,6 +302,27 @@ public class ExecutionCommand extends AgentCommand {
this.serviceType = serviceType;
}
+ /**
+ * Get a value indicating whether this service is enabled
+ * for credential store use.
+ *
+ * @return "true" or "false", any other value is
+ * considered as "false"
+ */
+ public String getCredentialStoreEnabled() {
+ return credentialStoreEnabled;
+ }
+
+ /**
+ * Set a value indicating whether this service is enabled
+ * for credential store use.
+ *
+ * @param credentialStoreEnabled
+ */
+ public void setCredentialStoreEnabled(String credentialStoreEnabled) {
+ this.credentialStoreEnabled = credentialStoreEnabled;
+ }
+
public String getComponentName() {
return componentName;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
index 75bef30..a25b875 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
@@ -329,7 +329,7 @@ public class HeartBeatHandler {
case BACKGROUND_EXECUTION_COMMAND:
case EXECUTION_COMMAND: {
ExecutionCommand ec = (ExecutionCommand)ac;
- LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, taskId {}",
+ LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, task ID {}",
ec.getHostname(), ec.getRole(), ec.getRoleCommand(), ec.getCommandId(), ec.getTaskId());
Map<String, String> hlp = ec.getHostLevelParams();
if (hlp != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index c3cd82e..9bf046b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2139,6 +2139,10 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
execCmd.setConfigurationAttributes(configurationAttributes);
execCmd.setConfigurationTags(configTags);
+ // Get the value of credential store enabled from the DB
+ Service clusterService = cluster.getService(serviceName);
+ execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));
+
// Create a local copy for each command
Map<String, String> commandParams = new TreeMap<String, String>();
if (commandParamsInp != null) { // if not defined
@@ -2345,6 +2349,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
execCmd.setAvailableServicesFromServiceInfoMap(ambariMetaInfo.getServices(stackId.getStackName(), stackId.getStackVersion()));
+
if ((execCmd != null) && (execCmd.getConfigurationTags().containsKey("cluster-env"))) {
LOG.debug("AmbariManagementControllerImpl.createHostAction: created ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, with cluster-env tags {}",
execCmd.getHostname(), execCmd.getRole(), execCmd.getRoleCommand(), execCmd.getCommandId(), execCmd.getConfigurationTags().get("cluster-env").get("tag"));
http://git-wip-us.apache.org/repos/asf/ambari/blob/24ac5cda/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index a50a116..ac58f64 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -48,6 +48,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -171,22 +172,31 @@ public class TestHeartbeatHandler {
ActionManager am = actionManagerTestHelper.getMockActionManager();
expect(am.getTasks(anyObject(List.class))).andReturn(new ArrayList<HostRoleCommand>());
replay(am);
+
+ Cluster cluster = heartbeatTestHelper.getDummyCluster();
+ Service hdfs = cluster.addService(HDFS);
+ hdfs.addServiceComponent(DATANODE);
+ hdfs.addServiceComponent(NAMENODE);
+ hdfs.addServiceComponent(SECONDARY_NAMENODE);
+ Collection<Host> hosts = cluster.getHosts();
+ assertEquals(hosts.size(), 1);
+
Clusters fsm = clusters;
- fsm.addHost(DummyHostname1);
- Host hostObject = clusters.getHost(DummyHostname1);
+ Host hostObject = hosts.iterator().next();
hostObject.setIPv4("ipv4");
hostObject.setIPv6("ipv6");
hostObject.setOsType(DummyOsType);
+ String hostname = hostObject.getHostName();
ActionQueue aq = new ActionQueue();
HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector);
Register reg = new Register();
HostInfo hi = new HostInfo();
- hi.setHostName(DummyHostname1);
+ hi.setHostName(hostname);
hi.setOS(DummyOs);
hi.setOSRelease(DummyOSRelease);
- reg.setHostname(DummyHostname1);
+ reg.setHostname(hostname);
reg.setHardwareProfile(hi);
reg.setAgentVersion(metaInfo.getServerVersion());
handler.handleRegistration(reg);
@@ -195,19 +205,21 @@ public class TestHeartbeatHandler {
ExecutionCommand execCmd = new ExecutionCommand();
execCmd.setRequestAndStage(2, 34);
- execCmd.setHostname(DummyHostname1);
- aq.enqueue(DummyHostname1, new ExecutionCommand());
+ execCmd.setHostname(hostname);
+ execCmd.setClusterName(cluster.getClusterName());
+ execCmd.setServiceName(HDFS);
+ aq.enqueue(hostname, execCmd);
HeartBeat hb = new HeartBeat();
hb.setResponseId(0);
HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus);
List<Alert> al = new ArrayList<Alert>();
al.add(new Alert());
hb.setNodeStatus(hs);
- hb.setHostname(DummyHostname1);
+ hb.setHostname(hostname);
handler.handleHeartBeat(hb);
assertEquals(HostState.HEALTHY, hostObject.getState());
- assertEquals(0, aq.dequeueAll(DummyHostname1).size());
+ assertEquals(0, aq.dequeueAll(hostname).size());
}
[2/2] ambari git commit: AMBARI-19077: Ambari-server: Gather
dependent configuration types and password properties for a service component
Posted by sm...@apache.org.
AMBARI-19077: Ambari-server: Gather dependent configuration types and password properties for a service component
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d22422ba
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d22422ba
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d22422ba
Branch: refs/heads/branch-2.5
Commit: d22422ba18fff615416231574c45332a13c045d7
Parents: 24ac5cd
Author: Nahappan Somasundaram <ns...@hortonworks.com>
Authored: Sat Dec 3 10:37:53 2016 -0800
Committer: Nahappan Somasundaram <ns...@hortonworks.com>
Committed: Fri Dec 9 07:56:40 2016 -0800
----------------------------------------------------------------------
.../ambari_agent/CustomServiceOrchestrator.py | 156 +++++++++++++------
.../ambari/server/agent/ExecutionCommand.java | 33 ++++
.../AmbariManagementControllerImpl.java | 13 ++
.../ambari/server/state/ConfigHelper.java | 45 ++++++
.../ambari/server/state/PropertyInfo.java | 15 ++
5 files changed, 218 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d22422ba/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index f9ed4cf..5fd3068 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -72,6 +72,9 @@ class CustomServiceOrchestrator():
DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell'
+ # The property name used by the hadoop credential provider
+ CREDENTIAL_PROVIDER_PROPERTY_NAME = 'hadoop.security.credential.provider.path'
+
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
@@ -147,21 +150,97 @@ class CustomServiceOrchestrator():
return conf_dir
- def getAffectedConfigTypes(self, commandJson):
+ def getConfigTypeCredentials(self, commandJson):
"""
Gets the affected config types for the service in this command
+ with the password aliases and values.
+
+ Input:
+ {
+ "config-type1" : {
+ "password_key_name1":"password_value_name1",
+ "password_key_name2":"password_value_name2",
+ :
+ },
+ "config-type2" : {
+ "password_key_name1":"password_value_name1",
+ "password_key_name2":"password_value_name2",
+ :
+ },
+ :
+ }
+
+ Output:
+ {
+ "config-type1" : {
+ "alias1":"password1",
+ "alias2":"password2",
+ :
+ },
+ "config-type2" : {
+ "alias1":"password1",
+ "alias2":"password2",
+ :
+ },
+ :
+ }
+
+ If password_key_name is the same as password_value_name, then password_key_name is the password alias itself.
+ The value it points to is the password value.
+
+ If password_key_name is not the same as the password_value_name, then password_key_name points to the alias.
+ The value is pointed to by password_value_name.
+
+ For example:
+ Input:
+ {
+ "oozie-site" : {"oozie.service.JPAService.jdbc.password" : "oozie.service.JPAService.jdbc.password"},
+ "admin-properties" {"db_user":"db_password", "ranger.jpa.jdbc.credential.alias:ranger-admin-site" : "db_password"}
+ }
+
+ Output:
+ {
+ "oozie-site" : {"oozie.service.JPAService.jdbc.password" : "MyOozieJdbcPassword"},
+ "admin-properties" {"rangerdba" : "MyRangerDbaPassword", "rangeradmin":"MyRangerDbaPassword"},
+ }
:param commandJson:
:return:
"""
- return commandJson.get('configuration_attributes')
-
- def getCredentialProviderPropertyName(self):
- """
- Gets the property name used by the hadoop credential provider
- :return:
- """
- return 'hadoop.security.credential.provider.path'
+ configtype_credentials = {}
+ if 'configuration_credentials' in commandJson:
+ for config_type, password_properties in commandJson['configuration_credentials'].items():
+ if config_type in commandJson['configurations']:
+ value_names = []
+ config = commandJson['configurations'][config_type]
+ credentials = {}
+ for key_name, value_name in password_properties.items():
+ if key_name == value_name:
+ if value_name in config:
+ # password name is the alias
+ credentials[key_name] = config[value_name]
+ value_names.append(value_name) # Gather the value_name for deletion
+ else:
+ keyname_keyconfig = key_name.split(':')
+ key_name = keyname_keyconfig[0]
+ # if the key is in another configuration (cross reference),
+ # get the value of the key from that configuration
+ if (len(keyname_keyconfig) > 1):
+ if keyname_keyconfig[1] not in commandJson['configurations']:
+ continue
+ key_config = commandJson['configurations'][keyname_keyconfig[1]]
+ else:
+ key_config = config
+ if key_name in key_config and value_name in config:
+ # password name points to the alias
+ credentials[key_config[key_name]] = config[value_name]
+ value_names.append(value_name) # Gather the value_name for deletion
+ if len(credentials) > 0:
+ configtype_credentials[config_type] = credentials
+ for value_name in value_names:
+ # Remove the clear text password
+ config.pop(value_name, None)
+ return configtype_credentials
def generateJceks(self, commandJson):
"""
@@ -178,16 +257,6 @@ class CustomServiceOrchestrator():
logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
- # Password properties for a config type, if present,
- # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2}
- passwordProperties = {}
- config_types = self.getAffectedConfigTypes(commandJson)
- for config_type in config_types:
- elem = config_types.get(config_type)
- hidden = elem.get('hidden')
- if hidden is not None:
- passwordProperties[config_type] = hidden
-
# Set up the variables for the external command to generate a JCEKS file
java_home = commandJson['hostLevelParams']['java_home']
java_bin = '{java_home}/bin/java'.format(java_home=java_home)
@@ -196,31 +265,30 @@ class CustomServiceOrchestrator():
serviceName = commandJson['serviceName']
# Gather the password values and remove them from the configuration
- configs = commandJson.get('configurations')
- for key, value in passwordProperties.items():
- config = configs.get(key)
- if config is not None:
- file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key))
- if os.path.exists(file_path):
- os.remove(file_path)
- provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
- logger.info('provider_path={0}'.format(provider_path))
- for alias in value:
- pwd = config.get(alias)
- if pwd is not None:
- # Remove the clear text password
- config.pop(alias, None)
- # Add JCEKS provider path instead
- config[self.getCredentialProviderPropertyName()] = provider_path
- logger.debug("config={0}".format(config))
- protected_pwd = PasswordString(pwd)
- # Generate the JCEKS file
- cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
- alias, '-value', protected_pwd, '-provider', provider_path)
- logger.info(cmd)
- cmd_result = subprocess.call(cmd)
- logger.info('cmd_result = {0}'.format(cmd_result))
- os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
+ provider_paths = [] # A service may depend on multiple configs
+ configtype_credentials = self.getConfigTypeCredentials(commandJson)
+ for config_type, credentials in configtype_credentials.items():
+ config = commandJson['configurations'][config_type]
+ file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(config_type))
+ if os.path.exists(file_path):
+ os.remove(file_path)
+ provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
+ provider_paths.append(provider_path)
+ logger.info('provider_path={0}'.format(provider_path))
+ for alias, pwd in credentials.items():
+ logger.debug("config={0}".format(config))
+ protected_pwd = PasswordString(pwd)
+ # Generate the JCEKS file
+ cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
+ alias, '-value', protected_pwd, '-provider', provider_path)
+ logger.info(cmd)
+ cmd_result = subprocess.call(cmd)
+ logger.info('cmd_result = {0}'.format(cmd_result))
+ os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
+
+ if provider_paths:
+ # Add JCEKS provider paths instead
+ config[self.CREDENTIAL_PROVIDER_PROPERTY_NAME] = ','.join(provider_paths)
return cmd_result
http://git-wip-us.apache.org/repos/asf/ambari/blob/d22422ba/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index e46167a..5c4f08e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -119,6 +119,39 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("credentialStoreEnabled")
private String credentialStoreEnabled;
+ /**
+ * Map of config type to list of password properties
+ * <pre>
+ * {@code
+ * {
+ * "config_type1" :
+ * {
+ * "password_alias_name1:type1":"password_value_name1",
+ * "password_alias_name2:type2":"password_value_name2",
+ * :
+ * },
+ * "config_type2" :
+ * {
+ * "password_alias_name1:type1":"password_value_name1",
+ * "password_alias_name2:type2":"password_value_name2",
+ * :
+ * },
+ * :
+ * }
+ * }
+ * </pre>
+ */
+ @SerializedName("configuration_credentials")
+ private Map<String, Map<String, String>> configurationCredentials;
+
+ public void setConfigurationCredentials(Map<String, Map<String, String>> configurationCredentials) {
+ this.configurationCredentials = configurationCredentials;
+ }
+
+ public Map<String, Map<String, String>> getConfigurationCredentials() {
+ return this.configurationCredentials;
+ }
+
public String getCommandId() {
return commandId;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/d22422ba/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 9bf046b..b19a46c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -341,6 +341,8 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
@Inject
private AmbariActionExecutionHelper actionExecutionHelper;
+ private Map<String, Map<String, Map<String, String>>> configCredentialsForService = new HashMap<>();
+
@Inject
public AmbariManagementControllerImpl(ActionManager actionManager,
Clusters clusters, Injector injector) throws Exception {
@@ -2143,6 +2145,17 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
Service clusterService = cluster.getService(serviceName);
execCmd.setCredentialStoreEnabled(String.valueOf(clusterService.isCredentialStoreEnabled()));
+ // Get the map of service config type to password properties for the service
+ Map<String, Map<String, String>> configCredentials;
+ configCredentials = configCredentialsForService.get(clusterService.getName());
+ if (configCredentials == null) {
+ configCredentials = configHelper.getPropertiesWithPropertyType(stackId, clusterService,
+ PropertyType.PASSWORD);
+ configCredentialsForService.put(clusterService.getName(), configCredentials);
+ }
+
+ execCmd.setConfigurationCredentials(configCredentials);
+
// Create a local copy for each command
Map<String, String> commandParams = new TreeMap<String, String>();
if (commandParamsInp != null) { // if not defined
http://git-wip-us.apache.org/repos/asf/ambari/blob/d22422ba/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
index 8cf62ad..5f8beaf 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/ConfigHelper.java
@@ -41,6 +41,7 @@ import org.apache.ambari.server.orm.entities.ClusterConfigEntity;
import org.apache.ambari.server.state.PropertyInfo.PropertyType;
import org.apache.ambari.server.state.configgroup.ConfigGroup;
import org.apache.ambari.server.utils.SecretReference;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -488,6 +489,50 @@ public class ConfigHelper {
return result;
}
+ /**
+ * Gets a map of config types to password property names to password property value names.
+ *
+ * @param stackId
+ * @param service
+ * @param propertyType
+ * @return
+ * @throws AmbariException
+ */
+ public Map<String, Map<String, String>> getPropertiesWithPropertyType(StackId stackId, Service service, PropertyType propertyType)
+ throws AmbariException {
+ StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
+ Map<String, Map<String, String>> result = new HashMap<>();
+ Map<String, String> passwordProperties;
+ Set<PropertyInfo> serviceProperties = ambariMetaInfo.getServiceProperties(stack.getName(), stack.getVersion(), service.getName());
+ for (PropertyInfo serviceProperty : serviceProperties) {
+ if (serviceProperty.getPropertyTypes().contains(propertyType)) {
+ String stackPropertyConfigType = fileNameToConfigType(serviceProperty.getFilename());
+ passwordProperties = result.get(stackPropertyConfigType);
+ if (passwordProperties == null) {
+ passwordProperties = new HashMap<>();
+ result.put(stackPropertyConfigType, passwordProperties);
+ }
+ // If the password property is used by another property, it means the password property
+ // is a password value name while the use is the password alias name. If the user property
+ // is from another config type, include that in the password alias name as name:type.
+ if (serviceProperty.getUsedByProperties().size() > 0) {
+ for (PropertyDependencyInfo usedByProperty : serviceProperty.getUsedByProperties()) {
+ String propertyName = usedByProperty.getName();
+ if (!StringUtils.isEmpty(usedByProperty.getType())) {
+ propertyName += ':' + usedByProperty.getType();
+ }
+ passwordProperties.put(propertyName, serviceProperty.getName());
+ }
+ }
+ else {
+ passwordProperties.put(serviceProperty.getName(), serviceProperty.getName());
+ }
+ }
+ }
+
+ return result;
+ }
+
public Set<String> getPropertyValuesWithPropertyType(StackId stackId, PropertyType propertyType,
Cluster cluster, Map<String, DesiredConfig> desiredConfigs) throws AmbariException {
StackInfo stack = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
http://git-wip-us.apache.org/repos/asf/ambari/blob/d22422ba/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
index f89be4d..5881a10 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/PropertyInfo.java
@@ -76,6 +76,17 @@ public class PropertyInfo {
private Set<PropertyDependencyInfo> dependedByProperties =
new HashSet<PropertyDependencyInfo>();
+ /**
+ * The list of properties that use this property.
+ * Password properties may be used by other properties in
+ * the same config type or different config type, typically
+ * when asking for user name and password pairs.
+ */
+ @XmlElementWrapper(name="used-by")
+ @XmlElement(name = "property")
+ private Set<PropertyDependencyInfo> usedByProperties =
+ new HashSet<>();
+
//This method is called after all the properties (except IDREF) are unmarshalled for this object,
//but before this object is set to the parent object.
void afterUnmarshal(Unmarshaller unmarshaller, Object parent) {
@@ -96,6 +107,10 @@ public class PropertyInfo {
this.name = name;
}
+ public Set<PropertyDependencyInfo> getUsedByProperties() {
+ return usedByProperties;
+ }
+
public String getValue() {
return value;
}