You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/09/22 06:38:27 UTC
[2/2] ambari git commit: AMBARI-13094. Add Spark Thrift Ambari
Service (Judy Nash via smohanty)
AMBARI-13094. Add Spark Thrift Ambari Service (Judy Nash via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b550eaad
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b550eaad
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b550eaad
Branch: refs/heads/branch-2.1.2
Commit: b550eaad11c557d79f9afd3c1e04b220b63af252
Parents: e16b6bb
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Sep 21 21:36:15 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Sep 21 21:36:15 2015 -0700
----------------------------------------------------------------------
.../libraries/functions/hdp_select.py | 1 +
.../package/scripts/job_history_server.py | 4 +-
.../SPARK/1.2.0.2.2/package/scripts/params.py | 22 +-
.../1.2.0.2.2/package/scripts/service_check.py | 2 +-
.../1.2.0.2.2/package/scripts/setup_spark.py | 18 +-
.../1.2.0.2.2/package/scripts/spark_service.py | 51 +-
.../package/scripts/spark_thrift_server.py | 78 +++
.../1.2.0.2.2/package/scripts/status_params.py | 1 +
.../configurations/spark-hive-site-override.xml | 36 ++
.../configurations/spark-thrift-sparkconf.xml | 125 +++++
.../SPARK/1.4.1.2.3/metainfo.xml | 87 ++++
.../stacks/HDP/2.3/role_command_order.json | 3 +-
.../stacks/HDP/2.3/services/SPARK/metainfo.xml | 49 +-
.../2.3/SPARK/test_spark_thrift_server.py | 176 +++++++
.../stacks/2.3/configs/spark_default.json | 491 +++++++++++++++++++
15 files changed, 1115 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
index d0ee9ad..0c42823 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/hdp_select.py
@@ -58,6 +58,7 @@ SERVER_ROLE_DIRECTORY_MAP = {
'RANGER_ADMIN' : 'ranger-admin',
'RANGER_USERSYNC' : 'ranger-usersync',
'SPARK_JOBHISTORYSERVER' : 'spark-historyserver',
+ 'SPARK_THRIFTSERVER' : 'spark-thriftserver',
'NIMBUS' : 'storm-nimbus',
'SUPERVISOR' : 'storm-supervisor',
'HISTORYSERVER' : 'hadoop-mapreduce-historyserver',
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
index b3999c3..4923383 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
@@ -52,13 +52,13 @@ class JobHistoryServer(Script):
env.set_params(params)
self.configure(env)
- spark_service(action='start')
+ spark_service('jobhistoryserver', action='start')
def stop(self, env, rolling_restart=False):
import params
env.set_params(params)
- spark_service(action='stop')
+ spark_service('jobhistoryserver', action='stop')
def status(self, env):
import status_params
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index dfe5e7b..be81f59 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -23,7 +23,6 @@ import status_params
from setup_spark import *
-from resource_management import *
import resource_management.libraries.functions
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import hdp_select
@@ -38,7 +37,8 @@ from resource_management.libraries.script.script import Script
# for use with /usr/hdp/current/<component>
SERVER_ROLE_DIRECTORY_MAP = {
'SPARK_JOBHISTORYSERVER' : 'spark-historyserver',
- 'SPARK_CLIENT' : 'spark-client'
+ 'SPARK_CLIENT' : 'spark-client',
+ 'SPARK_THRIFTSERVER' : 'spark-thriftserver'
}
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SPARK_CLIENT")
@@ -72,7 +72,7 @@ if Script.is_hdp_stack_greater_or_equal("2.2"):
spark_pid_dir = status_params.spark_pid_dir
spark_home = format("/usr/hdp/current/{component_directory}")
-
+spark_thrift_server_conf_file = spark_conf + "/spark-thrift-sparkconf.conf"
java_home = config['hostLevelParams']['java_home']
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
@@ -84,10 +84,14 @@ spark_group = status_params.spark_group
user_group = status_params.user_group
spark_hdfs_user_dir = format("/user/{spark_user}")
spark_history_server_pid_file = status_params.spark_history_server_pid_file
+spark_thrift_server_pid_file = status_params.spark_thrift_server_pid_file
spark_history_server_start = format("{spark_home}/sbin/start-history-server.sh")
spark_history_server_stop = format("{spark_home}/sbin/stop-history-server.sh")
+spark_thrift_server_start = format("{spark_home}/sbin/start-thriftserver.sh")
+spark_thrift_server_stop = format("{spark_home}/sbin/stop-thriftserver.sh")
+
spark_submit_cmd = format("{spark_home}/bin/spark-submit")
spark_smoke_example = "org.apache.spark.examples.SparkPi"
spark_service_check_cmd = format(
@@ -133,10 +137,12 @@ kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executab
spark_kerberos_keytab = config['configurations']['spark-defaults']['spark.history.kerberos.keytab']
spark_kerberos_principal = config['configurations']['spark-defaults']['spark.history.kerberos.principal']
+# hive-site params
spark_hive_properties = {
'hive.metastore.uris': config['configurations']['hive-site']['hive.metastore.uris']
}
+# security settings
if security_enabled:
spark_principal = spark_kerberos_principal.replace('_HOST',spark_history_server_host.lower())
@@ -152,7 +158,15 @@ if security_enabled:
'hive.security.authorization.enabled': spark_hive_sec_authorization_enabled,
'hive.server2.enable.doAs': str(config['configurations']['hive-site']['hive.server2.enable.doAs']).lower()
})
-
+
+# thrift server support - available on HDP 2.3 or higher
+spark_thrift_sparkconf = None
+if version and compare_versions(format_hdp_stack_version(version), '2.3.2.0') >= 0 \
+ and 'spark-thrift-sparkconf' in config['configurations']:
+ spark_thrift_sparkconf = config['configurations']['spark-thrift-sparkconf']
+ if is_hive_installed:
+ spark_hive_properties.update(config['configurations']['spark-hive-site-override'])
+
default_fs = config['configurations']['core-site']['fs.defaultFS']
hdfs_site = config['configurations']['hdfs-site']
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
index b28782b..694f046 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
@@ -9,7 +9,7 @@ with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
+Unless required by applicable law or agree in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
index 5c01337..1044e6b 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
@@ -79,8 +79,16 @@ def setup_spark(env, type, action = None):
if params.is_hive_installed:
XmlConfig("hive-site.xml",
- conf_dir=params.spark_conf,
- configurations=params.spark_hive_properties,
- owner=params.spark_user,
- group=params.spark_group,
- mode=0644)
+ conf_dir=params.spark_conf,
+ configurations=params.spark_hive_properties,
+ owner=params.spark_user,
+ group=params.spark_group,
+ mode=0644)
+
+ # thrift server is not supported until HDP 2.3 or higher
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.3.0.0') >= 0 \
+ and 'spark-thrift-sparkconf' in params.config['configurations']:
+ PropertiesFile(params.spark_thrift_server_conf_file,
+ properties = params.config['configurations']['spark-thrift-sparkconf'],
+ key_value_delimiter = " ",
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
index d8b3e66..68a395b 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
@@ -25,9 +25,9 @@ from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
from resource_management.libraries.functions import format
from resource_management.core.resources.system import File, Execute
-def spark_service(action):
+def spark_service(name, action):
import params
-
+
if action == 'start':
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
@@ -40,18 +40,39 @@ def spark_service(action):
if resource_created:
params.HdfsResource(None, action="execute")
- no_op_test = format(
+ if name == 'jobhistoryserver':
+ historyserver_no_op_test = format(
'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
- Execute(format('{spark_history_server_start}'),
- user=params.spark_user,
- environment={'JAVA_HOME': params.java_home},
- not_if=no_op_test
- )
+ Execute(format('{spark_history_server_start}'),
+ user=params.spark_user,
+ environment={'JAVA_HOME': params.java_home},
+ not_if=historyserver_no_op_test)
+
+ elif name == 'sparkthriftserver':
+ thriftserver_no_op_test = format(
+ 'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
+ Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file}'),
+ user=params.spark_user,
+ environment={'JAVA_HOME': params.java_home},
+ not_if=thriftserver_no_op_test
+ )
elif action == 'stop':
- Execute(format('{spark_history_server_stop}'),
- user=params.spark_user,
- environment={'JAVA_HOME': params.java_home}
- )
- File(params.spark_history_server_pid_file,
- action="delete"
- )
\ No newline at end of file
+ if name == 'jobhistoryserver':
+ Execute(format('{spark_history_server_stop}'),
+ user=params.spark_user,
+ environment={'JAVA_HOME': params.java_home}
+ )
+ File(params.spark_history_server_pid_file,
+ action="delete"
+ )
+
+ elif name == 'sparkthriftserver':
+ Execute(format('{spark_thrift_server_stop}'),
+ user=params.spark_user,
+ environment={'JAVA_HOME': params.java_home}
+ )
+ File(params.spark_thrift_server_pid_file,
+ action="delete"
+ )
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py
new file mode 100644
index 0000000..170fbca
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_thrift_server.py
@@ -0,0 +1,78 @@
+#!/usr/bin/python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+import os
+
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.core.logger import Logger
+from resource_management.core import shell
+from setup_spark import setup_spark
+from spark_service import spark_service
+
+
+class SparkThriftServer(Script):
+
+ def install(self, env):
+ import params
+ env.set_params(params)
+
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ setup_spark(env, 'server', action = 'config')
+
+ def start(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+
+ self.configure(env)
+ spark_service('sparkthriftserver',action='start')
+
+ def stop(self, env, rolling_restart=False):
+ import params
+ env.set_params(params)
+ spark_service('sparkthriftserver',action='stop')
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ check_process_status(status_params.spark_thrift_server_pid_file)
+
+ def get_stack_to_component(self):
+ return {"HDP": "spark-thriftserver"}
+
+ def pre_rolling_restart(self, env):
+ import params
+
+ env.set_params(params)
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.3.2.0') >= 0:
+ conf_select.select(params.stack_name, "spark", params.version)
+ hdp_select.select("spark-thriftserver", params.version)
+
+if __name__ == "__main__":
+ SparkThriftServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/status_params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/status_params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/status_params.py
index ccd560e..bb0d35f 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/status_params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/status_params.py
@@ -29,3 +29,4 @@ user_group = config['configurations']['cluster-env']['user_group']
spark_pid_dir = config['configurations']['spark-env']['spark_pid_dir']
spark_history_server_pid_file = format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.deploy.history.HistoryServer-1.pid")
+spark_thrift_server_pid_file = format("{spark_pid_dir}/spark-{spark_user}-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid")
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-hive-site-override.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-hive-site-override.xml b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-hive-site-override.xml
new file mode 100644
index 0000000..2de64c5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-hive-site-override.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<configuration supports_final="true">
+ <property>
+ <name>hive.server2.thrift.port</name>
+ <value>10000</value>
+ <description>
+ TCP port number to listen on, default 10000.
+ </description>
+ </property>
+ <property>
+ <name>hive.server2.transport.mode</name>
+ <value>binary</value>
+ <description>
+ Expects one of [binary, http].
+ Transport mode of HiveServer2.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-thrift-sparkconf.xml
new file mode 100644
index 0000000..c42841f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/configurations/spark-thrift-sparkconf.xml
@@ -0,0 +1,125 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="true">
+
+ <property>
+ <name>spark.yarn.executor.memoryOverhead</name>
+ <value>384</value>
+ <description>
+ The amount of off heap memory (in megabytes) to be allocated per executor.
+ This is memory that accounts for things like VM overheads, interned strings,
+ other native overheads, etc.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.driver.memoryOverhead</name>
+ <value>384</value>
+ <description>
+ The amount of off heap memory (in megabytes) to be allocated per driver.
+ This is memory that accounts for things like VM overheads, interned strings,
+ other native overheads, etc.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.applicationMaster.waitTries</name>
+ <value>10</value>
+ <description>
+ Set the number of times the ApplicationMaster waits for the the Spark master and then
+ also the number of tries it waits for the SparkContext to be initialized.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.scheduler.heartbeat.interval-ms</name>
+ <value>5000</value>
+ <description>
+ The interval in ms in which the Spark application master heartbeats into the YARN ResourceManager.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.max.executor.failures</name>
+ <value>3</value>
+ <description>
+ The maximum number of executor failures before failing the application.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.queue</name>
+ <value>default</value>
+ <description>
+ The name of the YARN queue to which the application is submitted.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.containerLauncherMaxThreads</name>
+ <value>25</value>
+ <description>
+ The maximum number of threads to use in the application master for launching executor containers.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.submit.file.replication</name>
+ <value>3</value>
+ <description>
+ HDFS replication level for the files uploaded into HDFS for the application.
+ These include things like the Spark jar, the app jar, and any distributed cache files/archives.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.preserve.staging.files</name>
+ <value>false</value>
+ <description>
+ Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the
+ end of the job rather then delete them.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.driver.extraJavaOptions</name>
+ <value>-Dhdp.version={{hdp_full_version}}</value>
+ <description>
+ Specifies parameters that are passed to the JVM of the Spark driver.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.am.extraJavaOptions</name>
+ <value>-Dhdp.version={{hdp_full_version}}</value>
+ <description>
+ Specifies the parameters that are passed to the JVM of the Spark Application Master.
+ </description>
+ </property>
+
+ <property>
+ <name>spark.yarn.max.executor.failures</name>
+ <value>3</value>
+ <description>The maximum number of executor failures before failing the application.</description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
new file mode 100644
index 0000000..d44c46c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SPARK</name>
+ <extends>common-services/SPARK/1.3.1.2.3</extends>
+ <displayName>Spark</displayName>
+ <comment>Apache Spark is a fast and general engine for large-scale data processing.</comment>
+ <version>1.4.1.2.3</version>
+ <components>
+ <component>
+ <name>SPARK_THRIFTSERVER</name>
+ <displayName>Spark Thrift Server</displayName>
+ <category>MASTER</category>
+ <cardinality>0+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>YARN/YARN_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/spark_thrift_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <configuration-dependencies>
+ <config-type>spark-defaults</config-type>
+ <config-type>spark-env</config-type>
+ <config-type>spark-log4j-properties</config-type>
+ <config-type>spark-metrics-properties</config-type>
+ <config-type>spark-javaopts-properties</config-type>
+ <config-type>spark-thrift-sparkconf</config-type>
+ <config-type>spark-hive-site-override</config-type>
+ </configuration-dependencies>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
index b206fe4..9b1625d 100755
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/role_command_order.json
@@ -7,6 +7,7 @@
"RANGER_KMS_SERVER-START" : ["RANGER_ADMIN-START"],
"RANGER_KMS_SERVICE_CHECK-SERVICE_CHECK" : ["RANGER_KMS_SERVER-START"],
"PHOENIX_QUERY_SERVER-START": ["HBASE_MASTER-START"],
- "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"]
+ "ATLAS_SERVICE_CHECK-SERVICE_CHECK": ["ATLAS_SERVER-START"],
+ "SPARK_THRIFTSERVER-START" : ["NAMENODE-START"]
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
index 26c2dff..14161b4 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
@@ -23,7 +23,7 @@
<services>
<service>
<name>SPARK</name>
- <version>1.3.1.2.3</version>
+ <version>1.4.1.2.3</version>
<extends>common-services/SPARK/1.3.1.2.3</extends>
<osSpecifics>
<osSpecific>
@@ -52,6 +52,53 @@
<requiredServices>
<service>YARN</service>
</requiredServices>
+ <components>
+ <component>
+ <name>SPARK_THRIFTSERVER</name>
+ <displayName>Spark Thrift Server</displayName>
+ <deleted>true</deleted>
+ <category>MASTER</category>
+ <cardinality>0+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>YARN/YARN_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/spark_thrift_server.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+ <configuration-dependencies>
+ <config-type>spark-defaults</config-type>
+ <config-type>spark-env</config-type>
+ <config-type>spark-log4j-properties</config-type>
+ <config-type>spark-metrics-properties</config-type>
+ <config-type>spark-javaopts-properties</config-type>
+ <config-type>spark-thrift-sparkconf</config-type>
+ <config-type>spark-hive-site-override</config-type>
+ </configuration-dependencies>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b550eaad/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
new file mode 100644
index 0000000..a0b80f0
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
@@ -0,0 +1,176 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import json
+from mock.mock import MagicMock, patch
+from stacks.utils.RMFTestCase import *
+
+from only_for_platform import not_for_platform, PLATFORM_WINDOWS
+
+@not_for_platform(PLATFORM_WINDOWS)
+@patch("resource_management.libraries.functions.get_hdp_version", new=MagicMock(return_value="2.3.2.0-1597"))
+class TestSparkThriftServer(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
+ STACK_VERSION = "2.3"
+
+ @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+ def test_configure_default(self, copy_to_hdfs_mock):
+ copy_to_hdfs_mock = True
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_thrift_server.py",
+ classname = "SparkThriftServer",
+ command = "configure",
+ config_file="spark_default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default()
+ self.assertNoMoreResources()
+
+ @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+ def test_start_default(self, copy_to_hdfs_mock):
+ copy_to_hdfs_mock.return_value = False
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_thrift_server.py",
+ classname = "SparkThriftServer",
+ command = "start",
+ config_file="spark_default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default()
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-thriftserver.sh --properties-file /usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid` >/dev/null 2>&1',
+ user = 'spark',
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_thrift_server.py",
+ classname = "SparkThriftServer",
+ command = "stop",
+ config_file="spark_default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/stop-thriftserver.sh',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ user = 'spark',
+ )
+ self.assertResourceCalled('File', '/var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
+
+ def assert_configure_default(self):
+ self.assertResourceCalled('Directory', '/var/run/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/var/log/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('HdfsResource', '/user/spark',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ owner = 'spark',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ mode = 0775,
+ )
+ self.assertResourceCalled('HdfsResource', None,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = {u'a': u'b'},
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'],
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ )
+ self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/spark-client/conf/spark-defaults.conf',
+ key_value_delimiter = ' ',
+ properties = self.getConfig()['configurations']['spark-defaults'],
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/spark-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-env']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/log4j.properties',
+ content = '\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO',
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/metrics.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
+ content = ' -Dhdp.version=2.3.2.0-1597',
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf',
+ key_value_delimiter = ' ',
+ properties = self.getConfig()['configurations']['spark-thrift-sparkconf']
+ )
+
+ @patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
+ def test_pre_rolling_restart_23(self, copy_to_hdfs_mock):
+ config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+ version = '2.3.2.0-1234'
+ json_content['commandParams']['version'] = version
+
+ copy_to_hdfs_mock.return_value = True
+ mocks_dict = {}
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_thrift_server.py",
+ classname = "SparkThriftServer",
+ command = "pre_rolling_restart",
+ config_dict = json_content,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES,
+ call_mocks = [(0, None), (0, None)],
+ mocks_dict = mocks_dict)
+
+ self.assertResourceCalled('Execute', ('hdp-select', 'set', 'spark-thriftserver', version), sudo=True)
+ self.assertNoMoreResources()
+
+ self.assertEquals(1, mocks_dict['call'].call_count)
+ self.assertEquals(1, mocks_dict['checked_call'].call_count)
+ self.assertEquals(
+ ('conf-select', 'set-conf-dir', '--package', 'spark', '--stack-version', '2.3.2.0-1234', '--conf-version', '0'),
+ mocks_dict['checked_call'].call_args_list[0][0][0])
+ self.assertEquals(
+ ('conf-select', 'create-conf-dir', '--package', 'spark', '--stack-version', '2.3.2.0-1234', '--conf-version', '0'),
+ mocks_dict['call'].call_args_list[0][0][0])