You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/06/01 16:37:11 UTC

ambari git commit: AMBARI-11585. Ambaripreupload script should be modified after the HdfsResource changes (aonishuk)

Repository: ambari
Updated Branches:
  refs/heads/trunk 45d779031 -> 6ba3da22b


AMBARI-11585. Ambaripreupload script should be modified after the HdfsResource changes (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6ba3da22
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6ba3da22
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6ba3da22

Branch: refs/heads/trunk
Commit: 6ba3da22baee31aefa6ff98ba706d7233cd8dc68
Parents: 45d7790
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Mon Jun 1 17:36:55 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Mon Jun 1 17:36:55 2015 +0300

----------------------------------------------------------------------
 .../4.0.0.2.0/package/scripts/oozie_service.py  |   7 +-
 .../main/resources/scripts/Ambaripreupload.py   | 237 +++++++++----------
 2 files changed, 114 insertions(+), 130 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba3da22/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 6afd250..ce93947 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -91,7 +91,11 @@ def oozie_service(action = 'start', rolling_restart=False):
                 user = params.oozie_user,
         )
       
-      if WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
+      
+      if params.host_sys_prepped:
+        print "Skipping creation of oozie sharelib as host is sys prepped"
+        hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check
+      elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
         # check with webhdfs is much faster than executing hadoop fs -ls. 
         util = WebHDFSUtil(params.hdfs_site, params.oozie_user, params.security_enabled)
         list_status = util.run_command(params.hdfs_share_dir, 'GETFILESTATUS', method='GET', ignore_status_codes=['404'], assertable_result=False)
@@ -113,6 +117,7 @@ def oozie_service(action = 'start', rolling_restart=False):
                              recursive_chmod=True,
         )
         params.HdfsResource(None, action="execute")
+        
 
     # start oozie
     Execute( start_cmd, environment=environment, user = params.oozie_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/6ba3da22/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 911aed4..548983c 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -17,15 +17,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
  
 """
-# Before running the script please add a proper pythopath, by executing:
-# export PYTHONPATH=/usr/lib/python2.6/site-packages 
+import os
+import sys
+os.environ["PATH"] += os.pathsep + "/var/lib/ambari-agent"
+sys.path.append("/usr/lib/python2.6/site-packages")
+
 import glob
 from logging import thread
-import sys
-import os
 import re
+import hashlib
 import tempfile
 import time
+import functools
 from xml.dom import minidom
 from xml.dom.minidom import parseString
  
@@ -37,13 +40,10 @@ from resource_management.core.logger import Logger
 from resource_management.core.resources.system import Execute
 from resource_management.libraries.functions.default import default
 from resource_management.libraries.functions.format import format
-from resource_management.libraries.resources.copy_from_local import CopyFromLocal
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
 from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
 from resource_management import Script
  
- 
-__all__ = ["copy_tarballs_to_hdfs", ]
- 
 """
 This file provides helper methods needed for the versioning of RPMs. Specifically, it does dynamic variable
 interpretation to replace strings like {{ hdp_stack_version }}  where the value of the
@@ -53,11 +53,37 @@ It assumes that {{ hdp_stack_version }} is constructed as ${major.minor.patch.re
 E.g., 998.2.2.1.0-998
 Please note that "-${build_number}" is optional.
 """
+
+def getPropertyValueFromConfigXMLFile(xmlfile, name, defaultValue=None):
+  xmldoc = minidom.parse(xmlfile)
+  propNodes = [node.parentNode for node in xmldoc.getElementsByTagName("name") if node.childNodes[0].nodeValue == name]
+  if len(propNodes) > 0:
+    for node in propNodes[-1].childNodes:
+      if node.nodeName == "value":
+        if len(node.childNodes) > 0:
+          return node.childNodes[0].nodeValue
+        else:
+          return ''
+  return defaultValue
+
+def get_fs_root(fsdefaultName=None):
+  return getPropertyValueFromConfigXMLFile("/etc/hadoop/conf/core-site.xml", "fs.defaultFS")
+  if fsdefaultName is None:
+    fsdefaultName = "fake"
+   
+  while (not fsdefaultName.startswith("wasb://")):
+    fsdefaultName =  getPropertyValueFromConfigXMLFile("/etc/hadoop/conf/core-site.xml", "fs.defaultFS")
+    if fsdefaultName is None:
+      fsdefaultName = "fake"
+    time.sleep(10)
+  
+  return fsdefaultName
  
 # These values must be the suffix of the properties in cluster-env.xml
 TAR_SOURCE_SUFFIX = "_tar_source"
 TAR_DESTINATION_FOLDER_SUFFIX = "_tar_destination_folder"
-class params():
+
+class params:
   hdfs_user = "hdfs"
   mapred_user ="mapred"
   hadoop_bin_dir="/usr/hdp/current/hadoop-client/bin"
@@ -66,30 +92,24 @@ class params():
   security_enabled = False
   oozie_user = "oozie"
   execute_path = "/usr/hdp/current/hadoop-client/bin"
- 
-params = params()
- 
- 
-class HdfsDirectory(Resource):
-  action = ForcedListArgument()
- 
-  dir_name = ResourceArgument(default=lambda obj: obj.name)
-  owner = ResourceArgument()
-  group = ResourceArgument()
-  mode = ResourceArgument()
-  recursive_chown = BooleanArgument(default=False)
-  recursive_chmod = BooleanArgument(default=False)
- 
-  conf_dir = ResourceArgument()
-  security_enabled = BooleanArgument(default=False)
-  keytab = ResourceArgument()
-  kinit_path_local = ResourceArgument()
-  hdfs_user = ResourceArgument()
-  bin_dir = ResourceArgument(default="")
- 
-  #action 'create' immediately creates all pending directory in efficient manner
-  #action 'create_delayed' add directory to list of pending directories
-  actions = Resource.actions + ["create","create_delayed"]
+  ambari_libs_dir = "/var/lib/ambari-agent/lib"
+  hdfs_site = ConfigDictionary({'dfs.webhdfs.enabled':False, 
+                                'dfs.namenode.http-address': getPropertyValueFromConfigXMLFile("/etc/hadoop/conf/hdfs-site.xml", "dfs.namenode.http-address")
+  })
+  fs_default = get_fs_root()
+  
+  HdfsResource = functools.partial(
+    HdfsResource,
+    user=hdfs_user,
+    security_enabled = False,
+    keytab = None,
+    kinit_path_local = None,
+    hadoop_bin_dir = hadoop_bin_dir,
+    hadoop_conf_dir = hadoop_conf_dir,
+    principal_name = None,
+    hdfs_site = hdfs_site,
+    default_fs = fs_default
+  )
  
 def _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed):
   """
@@ -103,34 +123,15 @@ def _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
   Must kinit before calling this function.
   """
  
-  return_value = 1
-  if source_and_dest_pairs and len(source_and_dest_pairs) > 0:
-    return_value = 0
-    for (source, destination) in source_and_dest_pairs:
-      try:
-        destination_dir = os.path.dirname(destination)
- 
-        HdfsDirectory(destination_dir,
-                      action="create",
-                      owner=file_owner,
-                      mode=0555,
-                      conf_dir=params.hadoop_conf_dir,
-                      hdfs_user=params.hdfs_user,
-        )
- 
-        CopyFromLocal(source,
-                      mode=0444,
-                      owner=file_owner,
-                      group=group_owner,
-                      dest_dir=destination_dir,
-                      kinnit_if_needed=kinit_if_needed,
-                      hdfs_user=params.hdfs_user,
-                      hadoop_bin_dir=params.hadoop_bin_dir,
-                      hadoop_conf_dir=params.hadoop_conf_dir
-        )
-      except:
-        return_value = 1
-  return return_value
+  for (source, destination) in source_and_dest_pairs:
+    params.HdfsResource(destination,
+                  action="create_on_execute",
+                  type = 'file',
+                  mode=0444,
+                  owner=file_owner,
+                  group=group_owner,
+                  source=source,
+    )
  
  
 def copy_tarballs_to_hdfs(source, dest, hdp_select_component_name, component_user, file_owner, group_owner):
@@ -187,37 +188,9 @@ def copy_tarballs_to_hdfs(source, dest, hdp_select_component_name, component_use
             user=component_user,
             path='/bin'
     )
-
-
-  does_hdfs_file_exist_cmd = "fs -ls %s" % destination_file
-  does_hdfs_file_exist = False
-  try:
-    ExecuteHadoop(does_hdfs_file_exist_cmd,
-                  user=component_user,
-                  logoutput=True,
-                  conf_dir=params.hadoop_conf_dir,
-                  bin_dir=params.hadoop_bin_dir
-    )
-    does_hdfs_file_exist = True
-  except Fail:
-    pass
  
-  if not does_hdfs_file_exist:
-    source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
-    return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
-  return 1
- 
-def getPropertyValueFromConfigXMLFile(xmlfile, name, defaultValue=None):
-  xmldoc = minidom.parse(xmlfile)
-  propNodes = [node.parentNode for node in xmldoc.getElementsByTagName("name") if node.childNodes[0].nodeValue == name]
-  if len(propNodes) > 0:
-    for node in propNodes[-1].childNodes:
-      if node.nodeName == "value":
-        if len(node.childNodes) > 0:
-          return node.childNodes[0].nodeValue
-        else:
-          return ''
-  return defaultValue
+  source_and_dest_pairs = [(component_tar_source_file, destination_file), ]
+  return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
 
 # See if hdfs path prefix is provided on the command line. If yes, use that value, if no
 # use empty string as default.
@@ -226,63 +199,56 @@ if len(sys.argv) == 2:
     hdfs_path_prefix = sys.argv[1]
 
 hadoop_conf_dir = params.hadoop_conf_dir
-fsdefaultName =  getPropertyValueFromConfigXMLFile("/etc/hadoop/conf/core-site.xml", "fs.defaultFS")
-if fsdefaultName is None:
-  fsdefaultName = "fake"
- 
-while (not fsdefaultName.startswith("wasb://")):
-  fsdefaultName =  getPropertyValueFromConfigXMLFile("/etc/hadoop/conf/core-site.xml", "fs.defaultFS")
-  if fsdefaultName is None:
-    fsdefaultName = "fake"
-  time.sleep(10)
- 
-fs_root = fsdefaultName
  
 oozie_libext_dir = "/usr/hdp/current/oozie-server/libext"
 oozie_home="/usr/hdp/current/oozie-server"
+oozie_setup_sh="/usr/hdp/current/oozie-server/bin/oozie-setup.sh"
+oozie_tmp_dir = "/var/tmp/oozie"
 configure_cmds = []
 configure_cmds.append(('tar','-xvf', oozie_home + '/oozie-sharelib.tar.gz','-C', oozie_home))
 configure_cmds.append(('cp', "/usr/share/HDP-oozie/ext-2.2.zip", "/usr/hdp/current/oozie-server/libext"))
 configure_cmds.append(('chown', 'oozie:hadoop', oozie_libext_dir + "/ext-2.2.zip"))
  
 no_op_test = "ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1"
+
 with Environment() as env:
+  env.set_params(params)
+  
+  hashcode_file = format("{oozie_home}/.hashcode")
+  hashcode = hashlib.md5(format('{oozie_home}/oozie-sharelib.tar.gz')).hexdigest()
+  skip_recreate_sharelib = format("test -f {hashcode_file} && test -d {oozie_home}/share && [[ `cat {hashcode_file}` == '{hashcode}' ]]")
+
   Execute( configure_cmds,
-           not_if  = no_op_test,
+           not_if  = format("{no_op_test} || {skip_recreate_sharelib}"), 
            sudo = True,
            )
+  Execute(format("cd {oozie_tmp_dir} && {oozie_setup_sh} prepare-war"),
+    user = params.oozie_user,
+    not_if  = format("{no_op_test} || {skip_recreate_sharelib}")
+  )
+  File(hashcode_file,
+       content = hashcode,
+       mode = 0644,
+  )
 
   oozie_shared_lib = format("/usr/hdp/current/oozie-server/share")
   oozie_user = 'oozie'
   oozie_hdfs_user_dir = format("{hdfs_path_prefix}/user/{oozie_user}")
   kinit_if_needed = ''
 
-  #Ideally, we would want to run: put_shared_lib_to_hdfs_cmd = format("{oozie_setup_sh} sharelib create -fs {fs_root} -locallib {oozie_shared_lib}")
-  #However given that oozie_setup_sh does not support an arbitrary hdfs path prefix, we are simulating the same command below
-  put_shared_lib_to_hdfs_cmd = format("hadoop --config {hadoop_conf_dir} dfs -copyFromLocal {oozie_shared_lib}/lib/** {oozie_hdfs_user_dir}/share/lib/lib_20150212065327")
-
-  oozie_cmd = format("{put_shared_lib_to_hdfs_cmd} ; hadoop --config {hadoop_conf_dir} dfs -chmod -R 755 {oozie_hdfs_user_dir}/share")
-
-  #Check if destination folder already exists
-  does_hdfs_file_exist_cmd = "fs -ls %s" % format("{oozie_hdfs_user_dir}/share")
-  try:
-    ExecuteHadoop(does_hdfs_file_exist_cmd,
-                  user=oozie_user,
-                  logoutput=True,
-                  conf_dir=params.hadoop_conf_dir,
-                  bin_dir=params.hadoop_bin_dir
-    )
-  except Fail:
-    #If dir does not exist create it and put files there
-    HdfsDirectory(format("{oozie_hdfs_user_dir}/share/lib/lib_20150212065327"),
-                  action="create",
-                  owner=oozie_user,
-                  mode=0555,
-                  conf_dir=params.hadoop_conf_dir,
-                  hdfs_user=params.hdfs_user,
-                  )
-    Execute( oozie_cmd, user = params.oozie_user, not_if = None,
-             path = params.execute_path )
+  params.HdfsResource(format("{oozie_hdfs_user_dir}/share/"),
+    action="delete_on_execute",
+    type = 'directory'
+  )
+    
+  params.HdfsResource(format("{oozie_hdfs_user_dir}/share"),
+    action="create_on_execute",
+    type = 'directory',
+    mode=0755,
+    recursive_chmod = True,
+    owner=oozie_user,
+    source = oozie_shared_lib,
+  )
 
   copy_tarballs_to_hdfs("/usr/hdp/current/hadoop-client/mapreduce.tar.gz", hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
   copy_tarballs_to_hdfs("/usr/hdp/current/tez-client/lib/tez.tar.gz", hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/tez/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
@@ -290,3 +256,16 @@ with Environment() as env:
   copy_tarballs_to_hdfs("/usr/hdp/current/pig-client/pig.tar.gz", hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/pig/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
   copy_tarballs_to_hdfs("/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar", hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
   copy_tarballs_to_hdfs("/usr/hdp/current/sqoop-client/sqoop.tar.gz", hdfs_path_prefix+"/hdp/apps/{{ hdp_stack_version }}/sqoop/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
+  
+  
+  # jar shouldn't be used before (read comment below)
+  File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
+       mode=0644,
+       content=StaticFile("/var/lib/ambari-agent/cache/stacks/HDP/2.0.6/hooks/before-START/files/fast-hdfs-resource.jar")
+  )
+  # Create everything in one jar call (this is fast).
+  # (! Before everything should be executed with action="create_on_execute/delete_on_execute" for this time-optimization to work)
+  params.HdfsResource(None, 
+               logoutput=True,
+               action="execute"
+  )
\ No newline at end of file