You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2016/11/04 16:55:06 UTC
[25/50] [abbrv] ambari git commit: AMBARI-18700. Add HDFS resources
for HBase, Spark, Spark2,
Zeppelin to AmbariPreupload script. (Attila Doroszlai via stoader).
AMBARI-18700. Add HDFS resources for HBase, Spark, Spark2, Zeppelin to AmbariPreupload script. (Attila Doroszlai via stoader).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/be6b6b87
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/be6b6b87
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/be6b6b87
Branch: refs/heads/branch-dev-patch-upgrade
Commit: be6b6b872861ee4b1daa8f755f175996c95334f5
Parents: f1117c2
Author: Attila Doroszlai <ad...@hortonworks.com>
Authored: Thu Nov 3 11:53:05 2016 +0100
Committer: Toader, Sebastian <st...@hortonworks.com>
Committed: Thu Nov 3 11:55:03 2016 +0100
----------------------------------------------------------------------
.../2.0.0/package/scripts/spark_service.py | 9 +++++----
.../main/resources/scripts/Ambaripreupload.py | 19 ++++++++++++++++++-
2 files changed, 23 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/be6b6b87/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
index 1cbca8b..12988c8 100755
--- a/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK2/2.0.0/package/scripts/spark_service.py
@@ -57,10 +57,11 @@ def spark_service(name, upgrade_type=None, action=None):
if name == 'jobhistoryserver' and effective_version and check_stack_feature(StackFeature.SPARK_16PLUS, effective_version):
# create & copy spark2-hdp-yarn-archive.tar.gz to hdfs
- source_dir=params.spark_home+"/jars"
- tmp_archive_file=get_tarball_paths("spark2")[1]
- make_tarfile(tmp_archive_file, source_dir)
- copy_to_hdfs("spark2", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
+ if not params.sysprep_skip_copy_tarballs_hdfs:
+ source_dir=params.spark_home+"/jars"
+ tmp_archive_file=get_tarball_paths("spark2")[1]
+ make_tarfile(tmp_archive_file, source_dir)
+ copy_to_hdfs("spark2", params.user_group, params.hdfs_user, skip=params.sysprep_skip_copy_tarballs_hdfs)
# create spark history directory
params.HdfsResource(params.spark_history_dir,
type="directory",
http://git-wip-us.apache.org/repos/asf/ambari/blob/be6b6b87/ambari-server/src/main/resources/scripts/Ambaripreupload.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/Ambaripreupload.py b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
index 1082b5e..c04424f 100644
--- a/ambari-server/src/main/resources/scripts/Ambaripreupload.py
+++ b/ambari-server/src/main/resources/scripts/Ambaripreupload.py
@@ -243,6 +243,7 @@ with Environment() as env:
return _copy_files(source_and_dest_pairs, file_owner, group_owner, kinit_if_needed)
def createHdfsResources():
+ print "Creating hdfs directories..."
params.HdfsResource(format('{hdfs_path_prefix}/atshistory'), user='hdfs', change_permissions_for_parents=True, owner='yarn', group='hadoop', type='directory', action= ['create_on_execute'], mode=0755)
params.HdfsResource(format('{hdfs_path_prefix}/user/hcat'), owner='hcat', type='directory', action=['create_on_execute'], mode=0755)
params.HdfsResource(format('{hdfs_path_prefix}/hive/warehouse'), owner='hive', type='directory', action=['create_on_execute'], mode=0777)
@@ -261,8 +262,23 @@ with Environment() as env:
params.HdfsResource(format('{hdfs_path_prefix}/amshbase/staging'), owner='ams', type='directory', action=['create_on_execute'], mode=0711)
params.HdfsResource(format('{hdfs_path_prefix}/user/ams/hbase'), owner='ams', type='directory', action=['create_on_execute'], mode=0775)
params.HdfsResource(format('{hdfs_path_prefix}/hdp'), owner='hdfs', type='directory', action=['create_on_execute'], mode=0755)
+ params.HdfsResource(format('{hdfs_path_prefix}/user/spark'), owner='spark', group='hadoop', type='directory', action=['create_on_execute'], mode=0775)
+ params.HdfsResource(format('{hdfs_path_prefix}/user/livy'), owner='livy', group='hadoop', type='directory', action=['create_on_execute'], mode=0775)
params.HdfsResource(format('{hdfs_path_prefix}/hdp/spark-events'), owner='spark', group='hadoop', type='directory', action=['create_on_execute'], mode=0777)
+ params.HdfsResource(format('{hdfs_path_prefix}/hdp/spark2-events'), owner='spark', group='hadoop', type='directory', action=['create_on_execute'], mode=0777)
+ params.HdfsResource(format('{hdfs_path_prefix}/hbase'), owner='hbase', type='directory', action=['create_on_execute'])
+ params.HdfsResource(format('{hdfs_path_prefix}/apps/hbase/staging'), owner='hbase', type='directory', action=['create_on_execute'], mode=0711)
+ params.HdfsResource(format('{hdfs_path_prefix}/user/hbase'), owner='hbase', type='directory', action=['create_on_execute'], mode=0755)
+ params.HdfsResource(format('{hdfs_path_prefix}/apps/zeppelin'), owner='zeppelin', group='hadoop', type='directory', action=['create_on_execute'])
+ params.HdfsResource(format('{hdfs_path_prefix}/user/zeppelin'), owner='zeppelin', group='hadoop', type='directory', action=['create_on_execute'])
+ params.HdfsResource(format('{hdfs_path_prefix}/user/zeppelin/test'), owner='zeppelin', group='hadoop', type='directory', action=['create_on_execute'])
+ def copy_zeppelin_dependencies_to_hdfs(file_pattern):
+ spark_deps_full_path = glob.glob(file_pattern)
+ if spark_deps_full_path and os.path.exists(spark_deps_full_path[0]):
+ copy_tarballs_to_hdfs(spark_deps_full_path[0], hdfs_path_prefix+'/apps/zeppelin/', 'hadoop-mapreduce-historyserver', params.hdfs_user, 'zeppelin', 'zeppelin')
+ else:
+ Logger.info('zeppelin-spark-dependencies not found at %s.' % file_pattern)
def putCreatedHdfsResourcesToIgnore(env):
if not 'hdfs_files' in env.config:
@@ -397,8 +413,9 @@ with Environment() as env:
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/pig/pig.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/pig/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/hadoop-mapreduce/hadoop-streaming.jar"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/mapreduce/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
copy_tarballs_to_hdfs(format("/usr/hdp/{stack_version}/sqoop/sqoop.tar.gz"), hdfs_path_prefix+"/hdp/apps/{{ stack_version_formatted }}/sqoop/", 'hadoop-mapreduce-historyserver', params.mapred_user, params.hdfs_user, params.user_group)
- print "Creating hdfs directories..."
+
createHdfsResources()
+ copy_zeppelin_dependencies_to_hdfs(format("/usr/hdp/{stack_version}/zeppelin/interpreter/spark/dep/zeppelin-spark-dependencies-*.jar"))
putSQLDriverToOozieShared()
putCreatedHdfsResourcesToIgnore(env)