[ambari] branch trunk updated: AMBARI-23257. Address stack changes to create apps and user directories on nameservices (aonishuk)

[ao...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

aonishuk pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 6a10b04  AMBARI-23257. Address stack changes to create apps and user directories on nameservices (aonishuk)
6a10b04 is described below

commit 6a10b043bc37cb3790854af967066c1a26e93065
Author: Andrew Onishuk <ao...@hortonworks.com>
AuthorDate: Fri Mar 16 12:49:22 2018 +0200

    AMBARI-23257. Address stack changes to create apps and user directories on nameservices (aonishuk)
---
 .../libraries/functions/namenode_ha_utils.py       | 25 ++++++++++++++++------
 .../libraries/providers/hdfs_resource.py           | 16 ++++++++++----
 .../4.0.0.2.0/package/scripts/oozie_service.py     |  6 +++++-
 .../2.1.0.2.0/package/scripts/resourcemanager.py   |  6 +++++-
 4 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
index a786701..cea8a18 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/namenode_ha_utils.py
@@ -28,7 +28,7 @@ from resource_management.libraries.functions.hdfs_utils import is_https_enabled_
 
 
 __all__ = ["get_namenode_states", "get_active_namenode",
-           "get_property_for_active_namenode", "get_nameservices"]
+           "get_property_for_active_namenodes", "get_property_for_active_namenode", "get_nameservices"]
 
 HDFS_NN_STATE_ACTIVE = 'active'
 HDFS_NN_STATE_STANDBY = 'standby'
@@ -161,21 +161,34 @@ def get_active_namenode(hdfs_site, security_enabled, run_user):
     return active_namenodes[0]
 
   raise Fail('No active NameNode was found.')
+  
+def get_property_for_active_namenodes(hdfs_site, property_name, security_enabled, run_user):
+  """
+  Return format: {'ns1': 'value1', 'ns2', 'value2'}
+  """
+  result = {}
+
+  name_services = get_nameservices(hdfs_site)
+  for name_service in name_services:
+    result[name_service] = get_property_for_active_namenode(hdfs_site, name_service, property_name, security_enabled, run_user)
 
-def get_property_for_active_namenode(hdfs_site, property_name, security_enabled, run_user):
+  return result
+
+
+def get_property_for_active_namenode(hdfs_site, name_service, property_name, security_enabled, run_user):
   """
   For dfs.namenode.rpc-address:
     - In non-ha mode it will return hdfs_site[dfs.namenode.rpc-address]
     - In ha-mode it will return hdfs_site[dfs.namenode.rpc-address.nnha.nn2], where nnha is the name of HA, and nn2 is id of active NN
-    - In federated mode it fails since there is more than one active namenode
   """
   value = None
   rpc_key = None
   if _is_ha_config(hdfs_site):
     name_services = get_nameservices(hdfs_site)
-    if len(name_services) > 1:
-      raise Fail('Multiple name services not supported by this function')
-    name_service = name_services[0]
+
+    if name_service not in name_services:
+     raise Fail('Trying to get property for non-existing ns=\'{1}\'. Valid namespaces are {2}'.format(property_name, name_service, ','.join(name_services)))
+
     active_namenodes = get_namenode_states(hdfs_site, security_enabled, run_user)[0]
 
     if not len(active_namenodes):
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
index 22efd1d..57eff0b 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/hdfs_resource.py
@@ -149,10 +149,10 @@ class WebHDFSCallException(Fail):
     return None
 
 class WebHDFSUtil:
-  def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None):
-    https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address',
+  def __init__(self, hdfs_site, nameservice, run_user, security_enabled, logoutput=None):
+    https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, nameservice, 'dfs.namenode.https-address',
                                                                           security_enabled, run_user)
-    http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address',
+    http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, nameservice, 'dfs.namenode.http-address',
                                                                          security_enabled, run_user)
     self.is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
 
@@ -319,7 +319,15 @@ class HdfsResourceWebHDFS:
     if main_resource.resource.security_enabled:
       main_resource.kinit()
 
-    self.util = WebHDFSUtil(main_resource.resource.hdfs_site, main_resource.resource.user, 
+    nameservices = namenode_ha_utils.get_nameservices(main_resource.resource.hdfs_site)
+    if not nameservices:
+      self.action_delayed_for_nameservice(None, action_name, main_resource)
+    else:
+      for nameservice in nameservices:
+        self.action_delayed_for_nameservice(nameservice, action_name, main_resource)
+
+  def action_delayed_for_nameservice(self, nameservice, action_name, main_resource):
+    self.util = WebHDFSUtil(main_resource.resource.hdfs_site, nameservice, main_resource.resource.user,
                             main_resource.resource.security_enabled, main_resource.resource.logoutput)
     self.mode = oct(main_resource.resource.mode)[1:] if main_resource.resource.mode else main_resource.resource.mode
     self.mode_set = False
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 5fcbf45..29d6f4b 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -34,6 +34,7 @@ from resource_management.libraries.functions.show_logs import show_logs
 from resource_management.libraries.providers.hdfs_resource import WebHDFSUtil
 from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
 from ambari_commons import OSConst
+from resource_management.libraries.functions import namenode_ha_utils
 
 from resource_management.core import Logger
 
@@ -121,6 +122,9 @@ def oozie_service(action = 'start', upgrade_type=None):
                 user = params.oozie_user,
         )
 
+      nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site)
+      nameservice = None if not nameservices else nameservices[-1]
+
       if params.sysprep_skip_copy_oozie_share_lib_to_hdfs:
         Logger.info("Skipping creation of oozie sharelib as host is sys prepped")
         # Copy current hive-site to hdfs:/user/oozie/share/lib/spark/
@@ -137,7 +141,7 @@ def oozie_service(action = 'start', upgrade_type=None):
         hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check
       elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
         # check with webhdfs is much faster than executing hadoop fs -ls. 
-        util = WebHDFSUtil(params.hdfs_site, params.oozie_user, params.security_enabled)
+        util = WebHDFSUtil(params.hdfs_site, nameservice, params.oozie_user, params.security_enabled)
         list_status = util.run_command(params.hdfs_share_dir, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)
         hdfs_share_dir_exists = ('FileStatus' in list_status)
       else:
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
index 7885c94..601ced8 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/resourcemanager.py
@@ -38,6 +38,7 @@ from resource_management.libraries.providers.hdfs_resource import HdfsResourcePr
 from resource_management import is_empty
 from resource_management import shell
 from resource_management.core.resources.zkmigrator import ZkMigrator
+from resource_management.libraries.functions import namenode_ha_utils
 
 from yarn import yarn
 from service import service
@@ -226,9 +227,12 @@ class ResourcemanagerDefault(Resourcemanager):
 
       dir_exists = None
 
+      nameservices = namenode_ha_utils.get_nameservices(params.hdfs_site)
+      nameservice = None if not nameservices else nameservices[-1]
+      
       if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
         # check with webhdfs is much faster than executing hdfs dfs -test
-        util = WebHDFSUtil(params.hdfs_site, params.hdfs_user, params.security_enabled)
+        util = WebHDFSUtil(params.hdfs_site, nameservice, params.hdfs_user, params.security_enabled)
         list_status = util.run_command(dir_path, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)
         dir_exists = ('FileStatus' in list_status)
       else:

-- 
To stop receiving notification emails like this one, please contact
aonishuk@apache.org.