You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/12/02 20:26:41 UTC
[43/50] [abbrv] ambari git commit: Revert "AMBARI-18888:
Ambari-agent: Create configuration files with JCEKS information"
Revert "AMBARI-18888: Ambari-agent: Create configuration files with JCEKS information"
This reverts commit 615438b272e6bd8efd37481ef684ae7d68921e64.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/60a6bd45
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/60a6bd45
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/60a6bd45
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 60a6bd4575fb87fc26c4a277cbabf850ef2089e1
Parents: 32840c1
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Fri Dec 2 10:50:50 2016 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Fri Dec 2 10:50:50 2016 -0500
----------------------------------------------------------------------
ambari-agent/conf/unix/ambari-agent.ini | 3 -
.../ambari_agent/CustomServiceOrchestrator.py | 120 -------------------
ambari-agent/src/packages/tarball/all.xml | 30 -----
.../ambari/server/agent/ExecutionCommand.java | 28 -----
.../ambari/server/agent/HeartBeatHandler.java | 10 +-
.../server/agent/TestHeartbeatHandler.java | 28 ++---
6 files changed, 9 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-agent/conf/unix/ambari-agent.ini
----------------------------------------------------------------------
diff --git a/ambari-agent/conf/unix/ambari-agent.ini b/ambari-agent/conf/unix/ambari-agent.ini
index d6fcf5f..61948d4 100644
--- a/ambari-agent/conf/unix/ambari-agent.ini
+++ b/ambari-agent/conf/unix/ambari-agent.ini
@@ -46,9 +46,6 @@ keysdir=/var/lib/ambari-agent/keys
server_crt=ca.crt
passphrase_env_var_name=AMBARI_PASSPHRASE
ssl_verify_cert=0
-credential_lib_dir=/var/lib/ambari-agent/cred/lib
-credential_conf_dir=/var/lib/ambari-agent/cred/conf
-credential_shell_cmd=org.apache.hadoop.security.alias.CredentialShell
[services]
pidLookupPath=/var/run/
http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
index 1eab0a5..7d61611 100644
--- a/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
+++ b/ambari-agent/src/main/python/ambari_agent/CustomServiceOrchestrator.py
@@ -29,8 +29,6 @@ from FileCache import FileCache
from AgentException import AgentException
from PythonExecutor import PythonExecutor
from PythonReflectiveExecutor import PythonReflectiveExecutor
-from resource_management.core.utils import PasswordString
-import subprocess
import Constants
import hostname
@@ -67,11 +65,6 @@ class CustomServiceOrchestrator():
REFLECTIVELY_RUN_COMMANDS = FREQUENT_COMMANDS # -- commands which run a lot and often (this increases their speed)
DONT_BACKUP_LOGS_FOR_COMMANDS = FREQUENT_COMMANDS
- # Path where hadoop credential JARS will be available
- DEFAULT_CREDENTIAL_SHELL_LIB_PATH = '/var/lib/ambari-agent/cred/lib'
- DEFAULT_CREDENTIAL_CONF_DIR = '/var/lib/ambari-agent/cred/conf'
- DEFAULT_CREDENTIAL_SHELL_CMD = 'org.apache.hadoop.security.alias.CredentialShell'
-
def __init__(self, config, controller):
self.config = config
self.tmp_dir = config.get('agent', 'prefix')
@@ -85,14 +78,6 @@ class CustomServiceOrchestrator():
# cache reset will be called on every agent registration
controller.registration_listeners.append(self.file_cache.reset)
- # Construct the hadoop credential lib JARs path
- self.credential_shell_lib_path = os.path.join(config.get('security', 'credential_lib_dir',
- self.DEFAULT_CREDENTIAL_SHELL_LIB_PATH), '*')
-
- self.credential_conf_dir = config.get('security', 'credential_conf_dir', self.DEFAULT_CREDENTIAL_CONF_DIR)
-
- self.credential_shell_cmd = config.get('security', 'credential_shell_cmd', self.DEFAULT_CREDENTIAL_SHELL_CMD)
-
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -129,102 +114,6 @@ class CustomServiceOrchestrator():
else:
return PythonExecutor(self.tmp_dir, self.config)
- def getProviderDirectory(self, service_name):
- """
- Gets the path to the service conf folder where the JCEKS file will be created.
-
- :param service_name: Name of the service, for example, HIVE
- :return: lower case path to the service conf folder
- """
-
- # The stack definition scripts of the service can move the
- # JCEKS file around to where it wants, which is usually
- # /etc/<service_name>/conf
-
- conf_dir = os.path.join(self.credential_conf_dir, service_name.lower())
- if not os.path.exists(conf_dir):
- os.makedirs(conf_dir, 0644)
-
- return conf_dir
-
- def getAffectedConfigTypes(self, commandJson):
- """
- Gets the affected config types for the service in this command
-
- :param commandJson:
- :return:
- """
- return commandJson.get('configuration_attributes')
-
- def getCredentialProviderPropertyName(self):
- """
- Gets the property name used by the hadoop credential provider
- :return:
- """
- return 'hadoop.security.credential.provider.path'
-
- def generateJceks(self, commandJson):
- """
- Generates the JCEKS file with passwords for the service specified in commandJson
-
- :param commandJson: command JSON
- :return: An exit value from the external process that generated the JCEKS file. None if
- there are no passwords in the JSON.
- """
- cmd_result = None
- roleCommand = None
- if 'roleCommand' in commandJson:
- roleCommand = commandJson['roleCommand']
-
- logger.info('generateJceks: roleCommand={0}'.format(roleCommand))
-
- # Password properties for a config type, if present,
- # are under configuration_attributes:config_type:hidden:{prop1:attributes1, prop2, attributes2}
- passwordProperties = {}
- config_types = self.getAffectedConfigTypes(commandJson)
- for config_type in config_types:
- elem = config_types.get(config_type)
- hidden = elem.get('hidden')
- if hidden is not None:
- passwordProperties[config_type] = hidden
-
- # Set up the variables for the external command to generate a JCEKS file
- java_home = commandJson['hostLevelParams']['java_home']
- java_bin = '{java_home}/bin/java'.format(java_home=java_home)
-
- cs_lib_path = self.credential_shell_lib_path
- serviceName = commandJson['serviceName']
-
- # Gather the password values and remove them from the configuration
- configs = commandJson.get('configurations')
- for key, value in passwordProperties.items():
- config = configs.get(key)
- if config is not None:
- file_path = os.path.join(self.getProviderDirectory(serviceName), "{0}.jceks".format(key))
- if os.path.exists(file_path):
- os.remove(file_path)
- provider_path = 'jceks://file{file_path}'.format(file_path=file_path)
- logger.info('provider_path={0}'.format(provider_path))
- for alias in value:
- pwd = config.get(alias)
- if pwd is not None:
- # Remove the clear text password
- config.pop(alias, None)
- # Add JCEKS provider path instead
- config[self.getCredentialProviderPropertyName()] = provider_path
- logger.debug("config={0}".format(config))
- protected_pwd = PasswordString(pwd)
- # Generate the JCEKS file
- cmd = (java_bin, '-cp', cs_lib_path, self.credential_shell_cmd, 'create',
- alias, '-value', protected_pwd, '-provider', provider_path)
- logger.info(cmd)
- cmd_result = subprocess.call(cmd)
- logger.info('cmd_result = {0}'.format(cmd_result))
- os.chmod(file_path, 0644) # group and others should have read access so that the service user can read
-
- return cmd_result
-
-
def runCommand(self, command, tmpoutfile, tmperrfile, forced_command_name=None,
override_output_files=True, retry=False):
"""
@@ -290,15 +179,6 @@ class CustomServiceOrchestrator():
handle.on_background_command_started = self.map_task_to_process
del command['__handle']
- # If command contains credentialStoreEnabled, then
- # generate the JCEKS file for the configurations.
- credentialStoreEnabled = False
- if 'credentialStoreEnabled' in command:
- credentialStoreEnabled = (command['credentialStoreEnabled'] == "true")
-
- if credentialStoreEnabled == True:
- self.generateJceks(command)
-
json_path = self.dump_command_to_json(command, retry)
pre_hook_tuple = self.resolve_hook_script_path(hook_dir,
self.PRE_HOOK_PREFIX, command_name, script_type)
http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-agent/src/packages/tarball/all.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/src/packages/tarball/all.xml b/ambari-agent/src/packages/tarball/all.xml
index f8a54e3..c481208 100644
--- a/ambari-agent/src/packages/tarball/all.xml
+++ b/ambari-agent/src/packages/tarball/all.xml
@@ -190,34 +190,4 @@
<outputDirectory>/var/lib/${project.artifactId}/data</outputDirectory>
</file>
</files>
- <moduleSets>
- <moduleSet>
- <binaries>
- <includeDependencies>false</includeDependencies>
- <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
- <unpack>false</unpack>
- <directoryMode>755</directoryMode>
- <fileMode>644</fileMode>
- <dependencySets>
- <dependencySet>
- <outputDirectory>/var/lib/${project.artifactId}/cred/lib</outputDirectory>
- <unpack>false</unpack>
- <includes>
- <include>commons-cli:commons-cli</include>
- <include>commons-collections:commons-collections</include>
- <include>commons-configuration:commons-configuration</include>
- <include>commons-io:commons-io:jar:${commons.io.version}</include>
- <include>commons-lang:commons-lang</include>
- <include>commons-logging:commons-logging</include>
- <include>com.google.guava:guava</include>
- <include>org.slf4j:slf4j-api</include>
- <include>org.apache.hadoop:hadoop-common</include>
- <include>org.apache.hadoop:hadoop-auth</include>
- <include>org.apache.htrace:htrace-core</include>
- </includes>
- </dependencySet>
- </dependencySets>
- </binaries>
- </moduleSet>
- </moduleSets>
</assembly>
http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 7353366..29737ee 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -112,13 +112,6 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("availableServices")
private Map<String, String> availableServices = new HashMap<>();
- /**
- * "true" or "false" indicating whether this
- * service is enabled for credential store use.
- */
- @SerializedName("credentialStoreEnabled")
- private String credentialStoreEnabled;
-
public String getCommandId() {
return commandId;
}
@@ -302,27 +295,6 @@ public class ExecutionCommand extends AgentCommand {
this.serviceType = serviceType;
}
- /**
- * Get a value indicating whether this service is enabled
- * for credential store use.
- *
- * @return "true" or "false", any other value is
- * considered as "false"
- */
- public String getCredentialStoreEnabled() {
- return credentialStoreEnabled;
- }
-
- /**
- * Set a value indicating whether this service is enabled
- * for credential store use.
- *
- * @param credentialStoreEnabled
- */
- public void setCredentialStoreEnabled(String credentialStoreEnabled) {
- this.credentialStoreEnabled = credentialStoreEnabled;
- }
-
public String getComponentName() {
return componentName;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
index 48d11c4..75bef30 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/HeartBeatHandler.java
@@ -329,15 +329,7 @@ public class HeartBeatHandler {
case BACKGROUND_EXECUTION_COMMAND:
case EXECUTION_COMMAND: {
ExecutionCommand ec = (ExecutionCommand)ac;
- /*
- * Set the value of credentialStore enabled before sending the command to agent.
- */
- Cluster cluster = clusterFsm.getCluster(ec.getClusterName());
- Service service = cluster.getService(ec.getServiceName());
- if (service != null) {
- ec.setCredentialStoreEnabled(String.valueOf(service.isCredentialStoreEnabled()));
- }
- LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, task ID {}",
+ LOG.info("HeartBeatHandler.sendCommands: sending ExecutionCommand for host {}, role {}, roleCommand {}, and command ID {}, taskId {}",
ec.getHostname(), ec.getRole(), ec.getRoleCommand(), ec.getCommandId(), ec.getTaskId());
Map<String, String> hlp = ec.getHostLevelParams();
if (hlp != null) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/60a6bd45/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
index ac58f64..a50a116 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/agent/TestHeartbeatHandler.java
@@ -48,7 +48,6 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
@@ -172,31 +171,22 @@ public class TestHeartbeatHandler {
ActionManager am = actionManagerTestHelper.getMockActionManager();
expect(am.getTasks(anyObject(List.class))).andReturn(new ArrayList<HostRoleCommand>());
replay(am);
-
- Cluster cluster = heartbeatTestHelper.getDummyCluster();
- Service hdfs = cluster.addService(HDFS);
- hdfs.addServiceComponent(DATANODE);
- hdfs.addServiceComponent(NAMENODE);
- hdfs.addServiceComponent(SECONDARY_NAMENODE);
- Collection<Host> hosts = cluster.getHosts();
- assertEquals(hosts.size(), 1);
-
Clusters fsm = clusters;
- Host hostObject = hosts.iterator().next();
+ fsm.addHost(DummyHostname1);
+ Host hostObject = clusters.getHost(DummyHostname1);
hostObject.setIPv4("ipv4");
hostObject.setIPv6("ipv6");
hostObject.setOsType(DummyOsType);
- String hostname = hostObject.getHostName();
ActionQueue aq = new ActionQueue();
HeartBeatHandler handler = new HeartBeatHandler(fsm, aq, am, injector);
Register reg = new Register();
HostInfo hi = new HostInfo();
- hi.setHostName(hostname);
+ hi.setHostName(DummyHostname1);
hi.setOS(DummyOs);
hi.setOSRelease(DummyOSRelease);
- reg.setHostname(hostname);
+ reg.setHostname(DummyHostname1);
reg.setHardwareProfile(hi);
reg.setAgentVersion(metaInfo.getServerVersion());
handler.handleRegistration(reg);
@@ -205,21 +195,19 @@ public class TestHeartbeatHandler {
ExecutionCommand execCmd = new ExecutionCommand();
execCmd.setRequestAndStage(2, 34);
- execCmd.setHostname(hostname);
- execCmd.setClusterName(cluster.getClusterName());
- execCmd.setServiceName(HDFS);
- aq.enqueue(hostname, execCmd);
+ execCmd.setHostname(DummyHostname1);
+ aq.enqueue(DummyHostname1, new ExecutionCommand());
HeartBeat hb = new HeartBeat();
hb.setResponseId(0);
HostStatus hs = new HostStatus(Status.HEALTHY, DummyHostStatus);
List<Alert> al = new ArrayList<Alert>();
al.add(new Alert());
hb.setNodeStatus(hs);
- hb.setHostname(hostname);
+ hb.setHostname(DummyHostname1);
handler.handleHeartBeat(hb);
assertEquals(HostState.HEALTHY, hostObject.getState());
- assertEquals(0, aq.dequeueAll(hostname).size());
+ assertEquals(0, aq.dequeueAll(DummyHostname1).size());
}