You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by al...@apache.org on 2015/10/26 21:54:42 UTC
ambari git commit: AMBARI-13513. Add cmd option config to spark
thrift server (Judy Nash via alejandro)
Repository: ambari
Updated Branches:
refs/heads/trunk f8a9f63cc -> 7a0aa253d
AMBARI-13513. Add cmd option config to spark thrift server (Judy Nash via alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/7a0aa253
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/7a0aa253
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/7a0aa253
Branch: refs/heads/trunk
Commit: 7a0aa253d934e1b5a4997da9626ed9c8766a6a3e
Parents: f8a9f63
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Mon Oct 26 13:54:34 2015 -0700
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Mon Oct 26 13:54:34 2015 -0700
----------------------------------------------------------------------
.../SPARK/1.2.0.2.2/configuration/spark-env.xml | 6 ++++++
.../common-services/SPARK/1.2.0.2.2/package/scripts/params.py | 3 +++
.../SPARK/1.2.0.2.2/package/scripts/spark_service.py | 2 +-
.../test/python/stacks/2.3/SPARK/test_spark_thrift_server.py | 2 +-
.../src/test/python/stacks/2.3/configs/spark_default.json | 5 +++--
5 files changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0aa253/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml
index 79e3b52..25fcc6b 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-env.xml
@@ -100,4 +100,10 @@ fi
</value>
</property>
+ <property>
+ <name>spark_thrift_cmd_opts</name>
+ <description>additional spark thrift server commandline options</description>
+ <value>
+ </value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0aa253/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 518ba6d..c0cffbc 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -161,9 +161,12 @@ if security_enabled:
# thrift server support - available on HDP 2.3 or higher
spark_thrift_sparkconf = None
+spark_thrift_cmd_opts_properties = ''
if version and compare_versions(format_hdp_stack_version(version), '2.3.2.0') >= 0 \
and 'spark-thrift-sparkconf' in config['configurations']:
spark_thrift_sparkconf = config['configurations']['spark-thrift-sparkconf']
+ spark_thrift_cmd_opts_properties = config['configurations']['spark-env']['spark_thrift_cmd_opts']
+
if is_hive_installed:
spark_hive_properties.update(config['configurations']['spark-hive-site-override'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0aa253/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
index 68a395b..b274c61 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
@@ -51,7 +51,7 @@ def spark_service(name, action):
elif name == 'sparkthriftserver':
thriftserver_no_op_test = format(
'ls {spark_thrift_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_thrift_server_pid_file}` >/dev/null 2>&1')
- Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file}'),
+ Execute(format('{spark_thrift_server_start} --properties-file {spark_thrift_server_conf_file} {spark_thrift_cmd_opts_properties}'),
user=params.spark_user,
environment={'JAVA_HOME': params.java_home},
not_if=thriftserver_no_op_test
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0aa253/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
index b3b8235..ac07ffb 100644
--- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
@@ -53,7 +53,7 @@ class TestSparkThriftServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assert_configure_default()
- self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-thriftserver.sh --properties-file /usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf',
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-thriftserver.sh --properties-file /usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf --driver-memory 1g',
environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.sql.hive.thriftserver.HiveThriftServer2-1.pid` >/dev/null 2>&1',
user = 'spark',
http://git-wip-us.apache.org/repos/asf/ambari/blob/7a0aa253/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json b/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
index 9f3fb90..84cd31f 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/spark_default.json
@@ -21,7 +21,7 @@
"serviceName": "SLIDER",
"role": "SLIDER",
"commandParams": {
- "version": "2.3.1.0-2067",
+ "version": "2.3.2.0-2067",
"command_timeout": "300",
"service_package_folder": "OOZIE",
"script_type": "PYTHON",
@@ -160,7 +160,8 @@
"spark_pid_dir": "/var/run/spark",
"spark_log_dir": "/var/log/spark",
"spark_group": "spark",
- "spark_user": "spark"
+ "spark_user": "spark",
+ "spark_thrift_cmd_opts": "--driver-memory 1g"
},
"spark-metrics-properties": {
"content": "\n# syntax: [instance].sink|source.[name].[options]=[value]\n\n# This file configures Spark's internal metrics system. The metrics system is\n# divided into instances which correspond to internal components.\n# Each instance can be configured to report its metrics to one or more sinks.\n# Accepted values for [instance] are \"master\", \"worker\", \"executor\", \"driver\",\n# and \"applications\". A wild card \"*\" can be used as an instance name, in\n# which case all instances will inherit the supplied property.\n#\n# Within an instance, a \"source\" specifies a particular set of grouped metrics.\n# there are two kinds of sources:\n# 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will\n# collect a Spark component's internal state. Each instance is paired with a\n# Spark source that is added automatically.\n# 2. Common sources, like JvmSource, which will collect low level state.\n# These can be added through configuration options and ar
e then loaded\n# using reflection.\n#\n# A \"sink\" specifies where metrics are delivered to. Each instance can be\n# assigned one or more sinks.\n#\n# The sink|source field specifies whether the property relates to a sink or\n# source.\n#\n# The [name] field specifies the name of source or sink.\n#\n# The [options] field is the specific property of this source or sink. The\n# source or sink is responsible for parsing this property.\n#\n# Notes:\n# 1. To add a new sink, set the \"class\" option to a fully qualified class\n# name (see examples below).\n# 2. Some sinks involve a polling period. The minimum allowed polling period\n# is 1 second.\n# 3. Wild card properties can be overridden by more specific properties.\n# For example, master.sink.console.period takes precedence over\n# *.sink.console.period.\n# 4. A metrics specific configuration\n# \"spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties\" should be\n# added to Java properties using -Dspark.metrics.conf=xxx if you wa
nt to\n# customize metrics system. You can also put the file in ${SPARK_HOME}/conf\n# and it will be loaded automatically.\n# 5. MetricsServlet is added by default as a sink in master, worker and client\n# driver, you can send http request \"/metrics/json\" to get a snapshot of all the\n# registered metrics in json format. For master, requests \"/metrics/master/json\" and\n# \"/metrics/applications/json\" can be sent seperately to get metrics snapshot of\n# instance master and applications. MetricsServlet may not be configured by self.\n#\n\n## List of available sinks and their properties.\n\n# org.apache.spark.metrics.sink.ConsoleSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n\n# org.apache.spark.metrics.sink.CSVSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n# directory /tmp Where to store CSV files\n\n# org.apache.spark.metrics.sink.GangliaSink\n# Name: Default: Description:\n#
host NONE Hostname or multicast group of Ganglia server\n# port NONE Port of Ganglia server(s)\n# period 10 Poll period\n# unit seconds Units of poll period\n# ttl 1 TTL of messages sent by Ganglia\n# mode multicast Ganglia network mode ('unicast' or 'multicast')\n\n# org.apache.spark.metrics.sink.JmxSink\n\n# org.apache.spark.metrics.sink.MetricsServlet\n# Name: Default: Description:\n# path VARIES* Path prefix from the web server root\n# sample false Whether to show entire set of samples for histograms ('false' or 'true')\n#\n# * Default path is /metrics/json for all instances except the master. The master has two paths:\n# /metrics/aplications/json # App information\n# /metrics/master/json # Master information\n\n# org.apache.spark.metrics.sink.GraphiteSink\n# Name: Default: Description:\n# host NONE Hostname of Graphite server\n# port NONE Port of Graphite server\n# period 10 Poll period\n# unit seconds Units of poll period\n# prefix EMPTY STRING Prefix to prepend to metric name
\n\n## Examples\n# Enable JmxSink for all instances by class name\n#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink\n\n# Enable ConsoleSink for all instances by class name\n#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink\n\n# Polling period for ConsoleSink\n#*.sink.console.period=10\n\n#*.sink.console.unit=seconds\n\n# Master instance overlap polling period\n#master.sink.console.period=15\n\n#master.sink.console.unit=seconds\n\n# Enable CsvSink for all instances\n#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink\n\n# Polling period for CsvSink\n#*.sink.csv.period=1\n\n#*.sink.csv.unit=minutes\n\n# Polling directory for CsvSink\n#*.sink.csv.directory=/tmp/\n\n# Worker instance overlap polling period\n#worker.sink.csv.period=10\n\n#worker.sink.csv.unit=minutes\n\n# Enable jvm source for instance master, worker, driver and executor\n#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#worker.source.jvm.class=org.apache.spark.metrics.source
.JvmSource\n\n#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource"