You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/09/14 18:32:36 UTC
[16/35] ambari git commit: AMBARI-18383 : Oozie spark failed due to
incomplete spark share-lib directory. (avijayan)
AMBARI-18383 : Oozie spark failed due to incomplete spark share-lib directory. (avijayan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1e21842e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1e21842e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1e21842e
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 1e21842e3ac9e2b6a4552cb0d58294a22d3c2e40
Parents: 0855174
Author: Aravindan Vijayan <av...@hortonworks.com>
Authored: Tue Sep 13 16:21:33 2016 -0700
Committer: Aravindan Vijayan <av...@hortonworks.com>
Committed: Tue Sep 13 16:21:33 2016 -0700
----------------------------------------------------------------------
.../4.0.0.2.0/package/scripts/oozie_service.py | 11 ++++
.../main/resources/scripts/Ambaripreupload.py | 54 ++++++++++++++++++--
2 files changed, 62 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e21842e/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 8d80638..4740131 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -122,6 +122,17 @@ def oozie_service(action = 'start', upgrade_type=None):
if params.host_sys_prepped:
print "Skipping creation of oozie sharelib as host is sys prepped"
+ # Copy current hive-site to hdfs:/user/oozie/share/lib/spark/
+ params.HdfsResource(format("{hdfs_share_dir}/lib/spark/hive-site.xml"),
+ action="create_on_execute",
+ type = 'file',
+ mode=0444,
+ owner=params.oozie_user,
+ group=params.user_group,
+ source=format("{hive_conf_dir}/hive-site.xml"),
+ )
+ params.HdfsResource(None, action="execute")
+
hdfs_share_dir_exists = True # skip time-expensive hadoop fs -ls check
elif WebHDFSUtil.is_webhdfs_available(params.is_webhdfs_enabled, params.default_fs):
# check with webhdfs is much faster than executing hadoop fs -ls.
http://git-wip-us.apache.org/repos/asf/ambari/blob/1e21842e/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 7c8c601..9492e17 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -37,7 +37,7 @@ from resource_management.core import shell
from resource_management.core.base import Resource, ForcedListArgument, ResourceArgument, BooleanArgument
from resource_management.core.exceptions import Fail
from resource_management.core.logger import Logger
-from resource_management.core.resources.system import Execute
+from resource_management.core.resources.system import Execute, Directory
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.oozie_prepare_war import prepare_war
@@ -327,7 +327,55 @@ with Environment() as env:
action="delete_on_execute",
type = 'directory'
)
-
+
+ spark_client_dir = format("/usr/hdp/{stack_version}/spark")
+
+ if os.path.exists(spark_client_dir):
+ # Rename /usr/hdp/{stack_version}/oozie/share/lib/spark to spark-orig
+ Execute(("mv",
+ format("{oozie_shared_lib}/lib/spark"),
+ format("{oozie_shared_lib}/lib/spark-orig")),
+ sudo=True)
+
+ # Create /usr/hdp/{stack_version}/oozie/share/lib/spark
+ Directory(format("{oozie_shared_lib}/lib/spark"),
+ owner = oozie_user,
+ create_parents = True
+ )
+
+ # Copy oozie-sharelib-spark from /usr/hdp/{stack_version}/oozie/share/lib/spark-orig to spark
+ Execute(format("cp -f {oozie_shared_lib}/lib/spark-orig/oozie-sharelib-spark*.jar {oozie_shared_lib}/lib/spark"),
+ user=oozie_user)
+
+ # Copy /usr/hdp/{stack_version}/spark-client/*.jar except spark-examples*.jar
+ Execute(format("cp -P {spark_client_dir}/lib/*.jar {oozie_shared_lib}/lib/spark"),
+ user=oozie_user)
+ Execute(format("find {oozie_shared_lib}/lib/spark/ -type l -delete"),
+ user=oozie_user)
+ try:
+ Execute(format("rm -f {oozie_shared_lib}/lib/spark/spark-examples*.jar"),
+ user=oozie_user)
+ except:
+ print "No spark-examples jar files found in Spark client lib."
+
+ # Copy /usr/hdp/{stack_version}/spark-client/python/lib/*.zip & *.jar to /usr/hdp/{stack_version}/oozie/share/lib/spark
+ Execute(format("cp -f {spark_client_dir}/python/lib/*.zip {oozie_shared_lib}/lib/spark"),
+ user=oozie_user)
+
+ try:
+ Execute(format("cp -f {spark_client_dir}/python/lib/*.jar {oozie_shared_lib}/lib/spark"),
+ user=oozie_user)
+ except:
+ print "No jar files found in Spark client python lib."
+
+ # Skipping this step since it might cause issues to automated scripts that rely on hdfs://user/oozie/share/lib
+ # Rename /usr/hdp/{stack_version}/oozie/share/lib to lib_ts
+ # millis = int(round(time.time() * 1000))
+ # Execute(("mv",
+ # format("{oozie_shared_lib}/lib"),
+ # format("{oozie_shared_lib}/lib_{millis}")),
+ # sudo=True)
+
params.HdfsResource(format("{oozie_hdfs_user_dir}/share"),
action="create_on_execute",
type = 'directory',
@@ -353,7 +401,7 @@ with Environment() as env:
createHdfsResources()
putSQLDriverToOozieShared()
putCreatedHdfsResourcesToIgnore(env)
-
+
# jar shouldn't be used before (read comment below)
File(format("{ambari_libs_dir}/fast-hdfs-resource.jar"),
mode=0644,