ambari git commit: AMBARI-20696: Skip calling stack selector, conf selector tools for Nifi, Streamline, Registry custom services in HDP cluster (Madhuvanthi Radhakrishnan via jluniya)

[jl...@apache.org]

Repository: ambari
Updated Branches:
  refs/heads/trunk aca12b7a3 -> 4f2523e7f


AMBARI-20696: Skip calling stack selector, conf selector tools for Nifi, Streamline, Registry custom services in HDP cluster (Madhuvanthi Radhakrishnan via jluniya)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4f2523e7
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4f2523e7
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4f2523e7

Branch: refs/heads/trunk
Commit: 4f2523e7f08fd81859831353a18ffb52ab0af8e9
Parents: aca12b7
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Wed Apr 12 23:42:17 2017 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Wed Apr 12 23:42:17 2017 -0700

----------------------------------------------------------------------
 .../libraries/functions/stack_select.py         |  5 +++
 .../libraries/functions/stack_tools.py          | 10 +++++
 .../libraries/functions/version_select_util.py  | 42 ++++++++++++++++++++
 .../ambari/server/agent/ExecutionCommand.java   | 11 +++++
 .../AmbariCustomCommandExecutionHelper.java     |  8 ++--
 .../AmbariManagementControllerImpl.java         |  1 +
 .../internal/ClientConfigResourceProvider.java  |  1 +
 .../scripts/shared_initialization.py            |  5 ++-
 .../AmbariCustomCommandExecutionHelperTest.java |  1 +
 .../AmbariManagementControllerTest.java         |  4 +-
 10 files changed, 81 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
index 79393b9..00127b1 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_select.py
@@ -123,6 +123,11 @@ def select_all(version_to_select):
   """
   stack_root = Script.get_stack_root()
   (stack_selector_name, stack_selector_path, stack_selector_package) = stack_tools.get_stack_tool(stack_tools.STACK_SELECTOR_NAME)
+  if stack_selector_path is None:
+    Logger.warning(format("Skipping executing \"stack select all\ as stack selector path is None"))
+    return
+
+
   # it's an error, but it shouldn't really stop anything from working
   if version_to_select is None:
     Logger.error(format("Unable to execute {stack_selector_name} after installing because there was no version specified"))

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
index 02ae62d..93ec0b7 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/stack_tools.py
@@ -32,6 +32,7 @@ from resource_management.core.utils import pad
 STACK_SELECTOR_NAME = "stack_selector"
 CONF_SELECTOR_NAME = "conf_selector"
 
+
 def get_stack_tool(name):
   """
   Give a tool selector name get the stack-specific tool name, tool path, tool package
@@ -41,8 +42,17 @@ def get_stack_tool(name):
   from resource_management.libraries.functions.default import default
   stack_tools = None
   stack_tools_config = default("/configurations/cluster-env/stack_tools", None)
+  stack_name = default("/hostLevelParams/stack_name", None)
+  service_name = default("/serviceName", None)
+
+  #Get version Advertised tag to decide whether or not to call the selector tools
+  is_version_advertised = default("/versionAdvertised", True)
   if stack_tools_config:
     stack_tools = json.loads(stack_tools_config)
+  if service_name is not None:
+    if not is_version_advertised:
+      Logger.warning(format("No \"stack selector tool\" returned as the component does not advertise a version"))
+      return (None, None, None)
 
   if not stack_tools or not name or name.lower() not in stack_tools:
     Logger.warning("Cannot find config for {0} stack tool in {1}".format(str(name), str(stack_tools)))

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
index ff00a1f..85fe807 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/version_select_util.py
@@ -79,6 +79,48 @@ def get_component_version(stack_name, component_name):
   return version
 
 
+def get_component_version_with_stack_selector(stack_selector_path, component_name):
+  """
+   For specific cases where we deal with HDP add on services from a management pack, the version
+   needs to be determined by using the specific stack selector itself.
+   :param stack_selector_path: /usr/bin/hdf-select
+   Comes from the service which calls for this function.
+   :param component_name: Component name as a string necessary to get the version
+   :return: Returns a string if found, e.g., 2.2.1.0-2175, otherwise, returns None
+   This function can be called by custom services, hence should not be removed
+  """
+  version = None
+  out = None
+  code = -1
+  if not stack_selector_path:
+    Logger.error("Stack selector path not provided")
+  elif not os.path.exists(stack_selector_path):
+    Logger.error("Stack selector path does not exist")
+  elif not component_name:
+    Logger.error("Component name not provided")
+  else:
+    tmpfile = tempfile.NamedTemporaryFile()
+
+    get_stack_comp_version_cmd = ""
+    try:
+      # This is necessary because Ubuntu returns "stdin: is not a tty", see AMBARI-8088
+      with open(tmpfile.name, 'r') as file:
+        get_stack_comp_version_cmd = '{0} status {1} > {2}' .format(stack_selector_path, component_name, tmpfile.name)
+        code, stdoutdata = shell.call(get_stack_comp_version_cmd, quiet=True)
+        out = file.read()
+
+      if code != 0 or out is None:
+        raise Exception("Code is nonzero or output is empty")
+
+      Logger.debug("Command: %s\nOutput: %s" % (get_stack_comp_version_cmd, str(out)))
+      matches = re.findall(r"([\d\.]+\-\d+)", out)
+      version = matches[0] if matches and len(matches) > 0 else None
+    except Exception, e:
+      Logger.error("Could not determine stack version for component %s by calling '%s'. Return Code: %s, Output: %s." %
+                   (component_name, get_stack_comp_version_cmd, str(code), str(out)))
+  return version
+
+
 def get_versions_from_stack_root(stack_root):
   """
   Given a stack install root, returns a list of stack versions currently installed.

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
index 8c726a0..95da25e 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/agent/ExecutionCommand.java
@@ -97,6 +97,9 @@ public class ExecutionCommand extends AgentCommand {
   @SerializedName("serviceName")
   private String serviceName;
 
+  @SerializedName("versionAdvertised")
+  private boolean versionAdvertised;
+
   @SerializedName("serviceType")
   private String serviceType;
 
@@ -327,6 +330,14 @@ public class ExecutionCommand extends AgentCommand {
     this.serviceName = serviceName;
   }
 
+  public boolean getVersionAdvertised() {
+    return versionAdvertised;
+  }
+
+  public void setVersionAdvertised(boolean versionAdvertised) {
+    this.versionAdvertised = versionAdvertised;
+  }
+
   public String getServiceType() {
 	return serviceType;
   }

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
index d5018f5..d5d7cf4 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelper.java
@@ -326,6 +326,9 @@ public class AmbariCustomCommandExecutionHelper {
     AmbariMetaInfo ambariMetaInfo = managementController.getAmbariMetaInfo();
     ServiceInfo serviceInfo = ambariMetaInfo.getService(
         stackId.getStackName(), stackId.getStackVersion(), serviceName);
+    ComponentInfo componentInfo = ambariMetaInfo.getComponent(
+            stackId.getStackName(), stackId.getStackVersion(),
+            serviceName, componentName);
     StackInfo stackInfo = ambariMetaInfo.getStack
        (stackId.getStackName(), stackId.getStackVersion());
 
@@ -432,6 +435,7 @@ public class AmbariCustomCommandExecutionHelper {
       hostLevelParams.put(NOT_MANAGED_HDFS_PATH_LIST, notManagedHdfsPathList);
 
       execCmd.setHostLevelParams(hostLevelParams);
+      execCmd.setVersionAdvertised(componentInfo.isVersionAdvertised());
 
       Map<String, String> commandParams = new TreeMap<>();
       if (additionalCommandParams != null) {
@@ -443,10 +447,6 @@ public class AmbariCustomCommandExecutionHelper {
       boolean isInstallCommand = commandName.equals(RoleCommand.INSTALL.toString());
       String commandTimeout = configs.getDefaultAgentTaskTimeout(isInstallCommand);
 
-      ComponentInfo componentInfo = ambariMetaInfo.getComponent(
-          stackId.getStackName(), stackId.getStackVersion(),
-          serviceName, componentName);
-
       if (serviceInfo.getSchemaVersion().equals(AmbariMetaInfo.SCHEMA_VERSION_2)) {
         // Service check command is not custom command
         CommandScriptDefinition script = componentInfo.getCommandScript();

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
index 751ce08..76c1167 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
@@ -2457,6 +2457,7 @@ public class AmbariManagementControllerImpl implements AmbariManagementControlle
     hostParams.put(UNLIMITED_KEY_JCE_REQUIRED, (unlimitedKeyJCEPolicyRequired) ? "true" : "false");
 
     execCmd.setHostLevelParams(hostParams);
+    execCmd.setVersionAdvertised(componentInfo.isVersionAdvertised());
 
     Map<String, String> roleParams = new TreeMap<>();
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
index e42bd45..77b027c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/ClientConfigResourceProvider.java
@@ -443,6 +443,7 @@ public class ClientConfigResourceProvider extends AbstractControllerResourceProv
         jsonContent.put("hostLevelParams", hostLevelParams);
         jsonContent.put("hostname", hostName);
         jsonContent.put("clusterName", cluster.getClusterName());
+        jsonContent.put("versionAdvertised", componentInfo.isVersionAdvertised());
         jsonConfigurations = gson.toJson(jsonContent);
 
         File tmpDirectory = new File(TMP_PATH);

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
index 1609050..a0f58cb 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -22,6 +22,7 @@ import os
 from resource_management.libraries.functions import stack_tools
 from resource_management.libraries.functions.version import compare_versions
 from resource_management.core.resources.packaging import Package
+from resource_management.core.logger import Logger
 
 def install_packages():
   import params
@@ -31,7 +32,9 @@ def install_packages():
   packages = ['unzip', 'curl']
   if params.stack_version_formatted != "" and compare_versions(params.stack_version_formatted, '2.2') >= 0:
     stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME)
-    packages.append(stack_selector_package)
+    if stack_selector_package:
+      packages.append(stack_selector_package)
+
   Package(packages,
           retry_on_repo_unavailability=params.agent_stack_retry_on_unavailability,
           retry_count=params.agent_stack_retry_count)

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
index 71a02f5..4eca710 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariCustomCommandExecutionHelperTest.java
@@ -225,6 +225,7 @@ public class AmbariCustomCommandExecutionHelperTest {
     Assert.assertTrue(command.getHostLevelParams().containsKey(ExecutionCommand.KeyNames.USER_GROUPS));
     Assert.assertEquals("{\"zookeeperUser\":[\"zookeeperGroup\"]}", command.getHostLevelParams().get(ExecutionCommand.KeyNames.USER_GROUPS));
     Assert.assertEquals(true, command.getForceRefreshConfigTagsBeforeExecution());
+    Assert.assertFalse(command.getVersionAdvertised());
   }
 
   @Test

http://git-wip-us.apache.org/repos/asf/ambari/blob/4f2523e7/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 554e089..dfb8cb7 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -275,6 +275,7 @@ public class AmbariManagementControllerTest {
     hostDAO = injector.getInstance(HostDAO.class);
     topologyHostInfoDAO = injector.getInstance(TopologyHostInfoDAO.class);
     hostRoleCommandDAO = injector.getInstance(HostRoleCommandDAO.class);
+
     stackManagerMock = (StackManagerMock) ambariMetaInfo.getStackManager();
     EasyMock.replay(injector.getInstance(AuditLogger.class));
   }
@@ -1271,8 +1272,6 @@ public class AmbariManagementControllerTest {
     crReq.setDesiredConfig(Collections.singletonList(cr3));
     controller.updateClusters(Collections.singleton(crReq), null);
 
-
-
     // Install
     installService(cluster1, serviceName, false, false);
     ExecutionCommand ec =
@@ -1312,6 +1311,7 @@ public class AmbariManagementControllerTest {
     assertEquals("[\"myhdfsgroup\"]", ec.getHostLevelParams().get(ExecutionCommand.KeyNames.GROUP_LIST));
     assertTrue(ec.getHostLevelParams().containsKey(ExecutionCommand.KeyNames.USER_GROUPS));
     assertEquals("{\"myhdfsuser\":[\"myhdfsgroup\"]}", ec.getHostLevelParams().get(ExecutionCommand.KeyNames.USER_GROUPS));
+    assertEquals(ec.getVersionAdvertised(), false);
   }
 
   @Test