You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/04/19 16:06:40 UTC
[1/2] ambari git commit: Revert "AMBARI-20696: Skip calling stack
selector, conf selector tools for Nifi, Streamline,
Registry custom services in HDP cluster (Madhuvanthi Radhakrishnan via
jluniya)"
Repository: ambari
Updated Branches:
refs/heads/trunk f516b6c5a -> 91d438697
Revert "AMBARI-20696: Skip calling stack selector, conf selector tools for Nifi, Streamline, Registry custom services in HDP cluster (Madhuvanthi Radhakrishnan via jluniya)"
This reverts commit 4f2523e7f08fd81859831353a18ffb52ab0af8e9.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/998e7644
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/998e7644
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/998e7644
Branch: refs/heads/trunk
Commit: 998e7644433fec0b7cd5356e8ee1ee052551cec4
Parents: f516b6c
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Wed Apr 19 08:55:58 2017 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Wed Apr 19 08:55:58 2017 -0700
----------------------------------------------------------------------
.../libraries/functions/stack_select.py | 5 ---
.../libraries/functions/stack_tools.py | 10 -----
.../libraries/functions/version_select_util.py | 42 --------------------
.../ambari/server/agent/ExecutionCommand.java | 11 -----
.../AmbariCustomCommandExecutionHelper.java | 8 ++--
.../AmbariManagementControllerImpl.java | 1 -
.../internal/ClientConfigResourceProvider.java | 1 -
.../scripts/shared_initialization.py | 5 +--
.../AmbariCustomCommandExecutionHelperTest.java | 1 -
.../AmbariManagementControllerTest.java | 4 +-
10 files changed, 7 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
index 00127b1..79393b9 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
@@ -123,11 +123,6 @@ def select_all(version_to_select):
"""
stack_root = Script.get_stack_root()
(stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
- if stack_selector_path is None:
- Logger.warning(format("Skipping executing \"stack select all\ as stack selector path is None"))
- return
-
-
# it's an error, but it shouldn't really stop anything from working
if version_to_select is None:
Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified"))
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
index 93ec0b7..02ae62d 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
@@ -32,7 +32,6 @@ from resource_management.core.utils import pad
STACK_SELECTOR_NAME = "stack_selector"
CONF_SELECTOR_NAME = "conf_selector"
-
def get_stack_tool(name):
"""
Give a tool selector name get the stack-specific tool name, tool path, tool package
@@ -42,17 +41,8 @@ def get_stack_tool(name):
from resource_management.libraries.functions.default import default
stack_tools = None
stack_tools_config = default("/configurations/cluster-env/stack_tools", None)
- stack_name = default("/hostLevelParams/stack_name", None)
- service_name = default("/serviceName", None)
-
- #Get version Advertised tag to decide whether or not to call the selector tools
- is_version_advertised = default("/versionAdvertised", True)
if stack_tools_config:
stack_tools = json.loads(stack_tools_config)
- if service_name is not None:
- if not is_version_advertised:
- Logger.warning(format("No \"stack selector tool\" returned as the component does not advertise a version"))
- return (None, None, None)
if not stack_tools or not name or name.lower() not in stack_tools:
Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools)))
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index 85fe807..ff00a1f 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -79,48 +79,6 @@ def get_component_version(stack_name, component_name):
return version
-def get_component_version_with_stack_selector(stack_selector_path, component_name):
- """
- For specific cases where we deal with HDP add on services from a management pack, the version
- needs to be determined by using the specific stack selector itself.
- :param stack_selector_path: /usr/bin/hdf-select
- Comes from the service which calls for this function.
- :param component_name: Component name as a string necessary to get the version
- :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
- This function can be called by custom services, hence should not be removed
- """
- version = None
- out = None
- code = -1
- if not stack_selector_path:
- Logger.error("Stack selector path not provided")
- elif not os.path.exists(stack_selector_path):
- Logger.error("Stack selector path does not exist")
- elif not component_name:
- Logger.error("Component name not provided")
- else:
- tmpfile = tempfile.NamedTemporaryFile()
-
- get_stack_comp_version_cmd = ""
- try:
- # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
- with open(tmpfile.name, 'r') as file:
- get_stack_comp_version_cmd = '{0} status {1} > {2}' .format(stack_selector_path, component_name, tmpfile.name)
- code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True)
- out = file.read()
-
- if code != 0 or out is None:
- raise Exception("Code is nonzero or output is empty")
-
- Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
- matches = re.findall(r"([\d\.]+\-\d+)", out)
- version = matches[0] if matches and len(matches) > 0 else None
- except Exception, e:
- Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
- (component_name, get_stack_comp_version_cmd, str(code), str(out)))
- return version
-
-
def get_versions_from_stack_root(stack_root):
"""
Given a stack install root, returns a list of stack versions currently installed.
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 95da25e..8c726a0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -97,9 +97,6 @@ public class ExecutionCommand extends AgentCommand {
@SerializedName("serviceName")
private String serviceName;
- @SerializedName("versionAdvertised")
- private boolean versionAdvertised;
-
@SerializedName("serviceType")
private String serviceType;
@@ -330,14 +327,6 @@ public class ExecutionCommand extends AgentCommand {
this.serviceName = serviceName;
}
- public boolean getVersionAdvertised() {
- return versionAdvertised;
- }
-
- public void setVersionAdvertised(boolean versionAdvertised) {
- this.versionAdvertised = versionAdvertised;
- }
-
public String getServiceType() {
return serviceType;
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index a493b94..17bc718 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -326,9 +326,6 @@ public class AmbariCustomCommandExecutionHelper {
AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo();
ServiceInfo serviceInfo = ambariMetaInfo.getService(
stackId.getStackName(), stackId.getStackVersion(), serviceName);
- ComponentInfo componentInfo = ambariMetaInfo.getComponent(
- stackId.getStackName(), stackId.getStackVersion(),
- serviceName, componentName);
StackInfo stackInfo = ambariMetaInfo.getStack
(stackId.getStackName(), stackId.getStackVersion());
@@ -435,7 +432,6 @@ public class AmbariCustomCommandExecutionHelper {
hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
execCmd.setHostLevelParams(hostLevelParams);
- execCmd.setVersionAdvertised(componentInfo.isVersionAdvertised());
Map<String, String> commandParams = new TreeMap<>();
if (additionalCommandParams != null) {
@@ -447,6 +443,10 @@ public class AmbariCustomCommandExecutionHelper {
boolean isInstallCommand = commandName.equals(RoleCommand.INSTALL.toString());
int commandTimeout = Short.valueOf(configs.getDefaultAgentTaskTimeout(isInstallCommand)).intValue();
+ ComponentInfo componentInfo = ambariMetaInfo.getComponent(
+ stackId.getStackName(), stackId.getStackVersion(),
+ serviceName, componentName);
+
if (serviceInfo.getSchemaVersion().equals(AmbariMetaInfo.SCHEMA_VERSION_2)) {
// Service check command is not custom command
CommandScriptDefinition script = componentInfo.getCommandScript();
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 76c1167..751ce08 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2457,7 +2457,6 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
hostParams.put(UNLIMITED_KEY_JCE_REQUIRED, (unlimitedKeyJCEPolicyRequired) ? "true" : "false");
execCmd.setHostLevelParams(hostParams);
- execCmd.setVersionAdvertised(componentInfo.isVersionAdvertised());
Map<String, String> roleParams = new TreeMap<>();
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index 77b027c..e42bd45 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -443,7 +443,6 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
jsonContent.put("hostLevelParams", hostLevelParams);
jsonContent.put("hostname", hostName);
jsonContent.put("clusterName", cluster.getClusterName());
- jsonContent.put("versionAdvertised", componentInfo.isVersionAdvertised());
jsonConfigurations = gson.toJson(jsonContent);
File tmpDirectory = new File(TMP_PATH);
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
index a0f58cb..1609050 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -22,7 +22,6 @@ import os
from resource_management.libraries.functions import stack_tools
from resource_management.libraries.functions.version import compare_versions
from resource_management.core.resources.packaging import Package
-from resource_management.core.logger import Logger
def install_packages():
import params
@@ -32,9 +31,7 @@ def install_packages():
packages = ['unzip', 'curl']
if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
- if stack_selector_package:
- packages.append(stack_selector_package)
-
+ packages.append(stack_selector_package)
Package(packages,
retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
retry_count=params.agent_stack_retry_count)
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
index 4eca710..71a02f5 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
@@ -225,7 +225,6 @@ public class AmbariCustomCommandExecutionHelperTest {
Assert.assertTrue(command.getHostLevelParams().containsKey(ExecutionCommand.KeyNames.USER_GROUPS));
Assert.assertEquals("{\"zookeeperUser\":[\"zookeeperGroup\"]}", command.getHostLevelParams().get(ExecutionCommand.KeyNames.USER_GROUPS));
Assert.assertEquals(true, command.getForceRefreshConfigTagsBeforeExecution());
- Assert.assertFalse(command.getVersionAdvertised());
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/998e7644/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index dfb8cb7..554e089 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -275,7 +275,6 @@ public class AmbariManagementControllerTest {
hostDAO = injector.getInstance(HostDAO.class);
topologyHostInfoDAO = injector.getInstance(TopologyHostInfoDAO.class);
hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class);
-
stackManagerMock = (StackManagerMock) ambariMetaInfo.getStackManager();
EasyMock.replay(injector.getInstance(AuditLogger.class));
}
@@ -1272,6 +1271,8 @@ public class AmbariManagementControllerTest {
crReq.setDesiredConfig(Collections.singletonList(cr3));
controller.updateClusters(Collections.singleton(crReq), null);
+
+
// Install
installService(cluster1, serviceName, false, false);
ExecutionCommand ec =
@@ -1311,7 +1312,6 @@ public class AmbariManagementControllerTest {
assertEquals("[\"myhdfsgroup\"]", ec.getHostLevelParams().get(ExecutionCommand.KeyNames.GROUP_LIST));
assertTrue(ec.getHostLevelParams().containsKey(ExecutionCommand.KeyNames.USER_GROUPS));
assertEquals("{\"myhdfsuser\":[\"myhdfsgroup\"]}", ec.getHostLevelParams().get(ExecutionCommand.KeyNames.USER_GROUPS));
- assertEquals(ec.getVersionAdvertised(), false);
}
@Test
[2/2] ambari git commit: AMBARI-20696: Skip calling stack selector,
conf selector tools for Nifi, Streamline,
Registry custom services in HDP cluster (Madhuvanthi Radhakrishnan via
jluniya)
Posted by jl...@apache.org.
AMBARI-20696: Skip calling stack selector, conf selector tools for Nifi, Streamline, Registry custom services in HDP cluster (Madhuvanthi Radhakrishnan via jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/91d43869
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/91d43869
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/91d43869
Branch: refs/heads/trunk
Commit: 91d438697c902a2284e7e9c78e14995cead910e8
Parents: 998e764
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Wed Apr 19 09:06:33 2017 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Wed Apr 19 09:06:33 2017 -0700
----------------------------------------------------------------------
.../libraries/functions/version_select_util.py | 40 ++++++++++++++++++++
1 file changed, 40 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/91d43869/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index ff00a1f..79dc874 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -78,6 +78,46 @@ def get_component_version(stack_name, component_name):
return version
+def get_component_version_with_stack_selector(stack_selector_path, component_name):
+ """
+ For specific cases where we deal with HDP add on services from a management pack, the version
+ needs to be determined by using the specific stack selector itself.
+ :param stack_selector_path: /usr/bin/hdf-select
+ Comes from the service which calls for this function.
+ :param component_name: Component name as a string necessary to get the version
+ :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
+ This function can be called by custom services, hence should not be removed
+ """
+ version = None
+ out = None
+ code = -1
+ if not stack_selector_path:
+ Logger.error("Stack selector path not provided")
+ elif not os.path.exists(stack_selector_path):
+ Logger.error("Stack selector path does not exist")
+ elif not component_name:
+ Logger.error("Component name not provided")
+ else:
+ tmpfile = tempfile.NamedTemporaryFile()
+
+ get_stack_comp_version_cmd = ""
+ try:
+ # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
+ with open(tmpfile.name, 'r') as file:
+ get_stack_comp_version_cmd = '{0} status {1} > {2}' .format(stack_selector_path, component_name, tmpfile.name)
+ code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True)
+ out = file.read()
+
+ if code != 0 or out is None:
+ raise Exception("Code is nonzero or output is empty")
+
+ Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
+ matches = re.findall(r"([\d\.]+\-\d+)", out)
+ version = matches[0] if matches and len(matches) > 0 else None
+ except Exception, e:
+ Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
+ (component_name, get_stack_comp_version_cmd, str(code), str(out)))
+ return version
def get_versions_from_stack_root(stack_root):
"""