You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/03/03 18:41:10 UTC
ambari git commit: AMBARI-9811. Clean up spark stack (Gautam Borad
via smohanty)
Repository: ambari
Updated Branches:
refs/heads/branch-2.0.0 689893c8b -> 0191f6c58
AMBARI-9811. Clean up spark stack (Gautam Borad via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/0191f6c5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/0191f6c5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/0191f6c5
Branch: refs/heads/branch-2.0.0
Commit: 0191f6c588bce528ee7eca9a0a5c460981327e66
Parents: 689893c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Mar 3 09:40:57 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Mar 3 09:40:57 2015 -0800
----------------------------------------------------------------------
.../1.2.0.2.2/package/scripts/job_history_server.py | 15 +++------------
.../SPARK/1.2.0.2.2/package/scripts/setup_spark.py | 14 ++++++++------
.../SPARK/1.2.0.2.2/package/scripts/spark_client.py | 2 +-
.../resources/stacks/HDP/2.2/role_command_order.json | 1 +
4 files changed, 13 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/0191f6c5/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
index 05fd604..518352f 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
@@ -43,15 +43,12 @@ class JobHistoryServer(Script):
def install(self, env):
self.install_packages(env)
import params
-
env.set_params(params)
- self.configure(env)
def stop(self, env, rolling_restart=False):
import params
env.set_params(params)
- self.configure(env)
daemon_cmd = format('{spark_history_server_stop}')
Execute(daemon_cmd,
user=params.spark_user,
@@ -65,18 +62,12 @@ class JobHistoryServer(Script):
import params
env.set_params(params)
- self.configure(env)
+ setup_spark(env, 'server', action = 'start')
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
- # FIXME! TODO! remove this after soft link bug is fixed:
- #if not os.path.islink('/usr/hdp/current/spark'):
- # hdp_version = get_hdp_version()
- # cmd = 'ln -s /usr/hdp/' + hdp_version + '/spark /usr/hdp/current/spark'
- # Execute(cmd)
-
daemon_cmd = format('{spark_history_server_start}')
no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
@@ -94,12 +85,12 @@ class JobHistoryServer(Script):
# Recursively check all existing gmetad pid files
check_process_status(pid_file)
-
+ # Note: This function is not called from start()/install()
def configure(self, env):
import params
env.set_params(params)
- setup_spark(env)
+ setup_spark(env, 'server', action = 'config')
if __name__ == "__main__":
JobHistoryServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0191f6c5/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
index 8df5378..2c7d287 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
@@ -28,7 +28,7 @@ from resource_management.core.logger import Logger
from resource_management.core import shell
-def setup_spark(env):
+def setup_spark(env, type, action = None):
import params
env.set_params(params)
@@ -38,11 +38,13 @@ def setup_spark(env):
group=params.user_group,
recursive=True
)
- #params.HdfsDirectory(params.spark_hdfs_user_dir,
- # action="create",
- # owner=params.spark_user,
- # mode=0775
- #)
+ if type == 'server':
+ if action == 'start' or action == 'config':
+ params.HdfsDirectory(params.spark_hdfs_user_dir,
+ action="create",
+ owner=params.spark_user,
+ mode=0775
+ )
file_path = params.spark_conf + '/spark-defaults.conf'
create_file(file_path)
http://git-wip-us.apache.org/repos/asf/ambari/blob/0191f6c5/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
index b0cb39d..970dc6d 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
@@ -46,7 +46,7 @@ class SparkClient(Script):
import params
env.set_params(params)
- setup_spark(env)
+ setup_spark(env, 'client', action = 'config')
def status(self, env):
raise ClientComponentHasNoStatus()
http://git-wip-us.apache.org/repos/asf/ambari/blob/0191f6c5/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
index 928caa8..c86e2b1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/role_command_order.json
@@ -6,6 +6,7 @@
"RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_ADMIN-START"],
"RANGER_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_USERSYNC-START"],
"SPARK_SERVICE_CHECK-SERVICE_CHECK" : ["SPARK_JOBHISTORYSERVER-START", "APP_TIMELINE_SERVER-START"],
+ "SPARK_JOBHISTORYSERVER-START" : ["NAMENODE-START"],
"FALCON_SERVER-START": ["NAMENODE-START", "DATANODE-START", "OOZIE_SERVER-START"],
"WEBHCAT_SERVICE_CHECK-SERVICE_CHECK": ["WEBHCAT_SERVER-START"],
"FLUME_SERVICE_CHECK-SERVICE_CHECK": ["FLUME_HANDLER-START"],