You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2015/04/14 10:09:01 UTC
[2/2] ambari git commit: AMBARI-10461. Spark History Server does not
start with non-root agent (aonishuk)
AMBARI-10461. Spark History Server does not start with non-root agent (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/85acd9fa
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/85acd9fa
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/85acd9fa
Branch: refs/heads/trunk
Commit: 85acd9fa92ac6f2920ee2f5d1b5429832b3eeabc
Parents: ad1ad93
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Apr 14 11:08:49 2015 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Apr 14 11:08:49 2015 +0300
----------------------------------------------------------------------
.../libraries/functions/__init__.py | 1 +
.../libraries/providers/properties_file.py | 4 +-
.../libraries/resources/properties_file.py | 1 +
.../package/scripts/job_history_server.py | 71 ++----
.../SPARK/1.2.0.2.2/package/scripts/params.py | 18 +-
.../1.2.0.2.2/package/scripts/service_check.py | 32 +--
.../1.2.0.2.2/package/scripts/setup_spark.py | 145 +----------
.../1.2.0.2.2/package/scripts/spark_client.py | 21 +-
.../1.2.0.2.2/package/scripts/spark_service.py | 46 ++++
.../python/stacks/2.2/KNOX/test_knox_gateway.py | 21 +-
.../stacks/2.2/SLIDER/test_slider_client.py | 21 +-
.../stacks/2.2/SPARK/test_job_history_server.py | 255 +++++++++++++------
.../stacks/2.2/SPARK/test_spark_client.py | 120 +++++++++
.../2.2/SPARK/test_spark_service_check.py | 58 +++++
.../test/python/stacks/2.2/configs/default.json | 54 +++-
.../test/python/stacks/2.2/configs/secured.json | 35 ++-
16 files changed, 568 insertions(+), 335 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
index 6b3d572..f4822ce 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/__init__.py
@@ -40,6 +40,7 @@ from resource_management.libraries.functions.format_jvm_option import *
from resource_management.libraries.functions.constants import *
from resource_management.libraries.functions.get_hdp_version import *
from resource_management.libraries.functions.get_lzo_packages import *
+from resource_management.libraries.functions.dynamic_variable_interpretation import *
IS_WINDOWS = platform.system() == "Windows"
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py b/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py
index 8b2eba7..c1ea2c4 100644
--- a/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py
+++ b/ambari-common/src/main/python/resource_management/libraries/providers/properties_file.py
@@ -41,8 +41,8 @@ class PropertiesFileProvider(Provider):
config_content = InlineTemplate('''# Generated by Apache Ambari. {{time.asctime(time.localtime())}}
{% for key, value in properties_dict|dictsort %}
-{{key}}={{value}}{% endfor %}
- ''', extra_imports=[time], properties_dict=self.resource.properties)
+{{key}}{{key_value_delimiter}}{{value}}{% endfor %}
+ ''', extra_imports=[time], properties_dict=self.resource.properties, key_value_delimiter=self.resource.key_value_delimiter)
Logger.info(format("Generating properties file: {filepath}"))
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py b/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py
index 0e5afb4..527b0f6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py
+++ b/ambari-common/src/main/python/resource_management/libraries/resources/properties_file.py
@@ -33,5 +33,6 @@ class PropertiesFile(Resource):
mode = ResourceArgument()
owner = ResourceArgument()
group = ResourceArgument()
+ key_value_delimiter = ResourceArgument(default="=")
actions = Resource.actions + ["create"]
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
index ff04b58..353a1da 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/job_history_server.py
@@ -29,74 +29,53 @@ from resource_management.core.exceptions import ComponentIsNotRunning
from resource_management.core.logger import Logger
from resource_management.core import shell
from setup_spark import *
+from spark_service import spark_service
class JobHistoryServer(Script):
- def get_stack_to_component(self):
- return {"HDP": "spark-historyserver"}
-
- def pre_rolling_restart(self, env):
+ def install(self, env):
import params
-
env.set_params(params)
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set spark-historyserver {version}"))
- copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user, params.user_group)
-
- def install(self, env):
+
self.install_packages(env)
+
+ def configure(self, env):
import params
env.set_params(params)
-
- def stop(self, env, rolling_restart=False):
+
+ setup_spark(env, 'server', action = 'config')
+
+ def start(self, env, rolling_restart=False):
import params
-
env.set_params(params)
- daemon_cmd = format('{spark_history_server_stop}')
- Execute(daemon_cmd,
- user=params.spark_user,
- environment={'JAVA_HOME': params.java_home}
- )
- if os.path.isfile(params.spark_history_server_pid_file):
- os.remove(params.spark_history_server_pid_file)
-
+
+ self.configure(env)
+ spark_service(action='start')
- def start(self, env, rolling_restart=False):
+ def stop(self, env, rolling_restart=False):
import params
-
env.set_params(params)
- setup_spark(env, 'server', action='start')
-
- if params.security_enabled:
- spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
- Execute(spark_kinit_cmd, user=params.spark_user)
-
- copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user, params.user_group)
-
- daemon_cmd = format('{spark_history_server_start}')
- no_op_test = format(
- 'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
- Execute(daemon_cmd,
- user=params.spark_user,
- environment={'JAVA_HOME': params.java_home},
- not_if=no_op_test
- )
+
+ spark_service(action='stop')
def status(self, env):
import status_params
-
env.set_params(status_params)
- pid_file = format("{spark_history_server_pid_file}")
- # Recursively check all existing gmetad pid files
- check_process_status(pid_file)
- # Note: This function is not called from start()/install()
- def configure(self, env):
+ check_process_status(status_params.spark_history_server_pid_file)
+
+
+ def get_stack_to_component(self):
+ return {"HDP": "spark-historyserver"}
+
+ def pre_rolling_restart(self, env):
import params
env.set_params(params)
- setup_spark(env, 'server', action = 'config')
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set spark-historyserver {version}"))
+ copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user, params.user_group)
if __name__ == "__main__":
JobHistoryServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 8d2d285..5531e21 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -93,21 +93,7 @@ else:
spark_hive_sec_authorization_enabled = "false"
spark_yarn_historyServer_address = default(spark_history_server_host, "localhost")
-spark_yarn_applicationMaster_waitTries = default(
- "/configurations/spark-defaults/spark.yarn.applicationMaster.waitTries", '10')
-spark_yarn_submit_file_replication = default("/configurations/spark-defaults/spark.yarn.submit.file.replication", '3')
-spark_yarn_preserve_staging_files = default("/configurations/spark-defaults/spark.yarn.preserve.staging.files", "false")
-spark_yarn_scheduler_heartbeat_interval = default(
- "/configurations/spark-defaults/spark.yarn.scheduler.heartbeat.interval-ms", "5000")
-spark_yarn_queue = default("/configurations/spark-defaults/spark.yarn.queue", "default")
-spark_yarn_containerLauncherMaxThreads = default(
- "/configurations/spark-defaults/spark.yarn.containerLauncherMaxThreads", "25")
-spark_yarn_max_executor_failures = default("/configurations/spark-defaults/spark.yarn.max.executor.failures", "3")
-spark_yarn_executor_memoryOverhead = default("/configurations/spark-defaults/spark.yarn.executor.memoryOverhead", "384")
-spark_yarn_driver_memoryOverhead = default("/configurations/spark-defaults/spark.yarn.driver.memoryOverhead", "384")
-spark_history_provider = default("/configurations/spark-defaults/spark.history.provider",
- "org.apache.spark.deploy.yarn.history.YarnHistoryProvider")
-spark_history_ui_port = default("/configurations/spark-defaults/spark.history.ui.port", "18080")
+spark_history_ui_port = config['configurations']['spark-defaults']['spark.history.ui.port']
spark_env_sh = config['configurations']['spark-env']['content']
spark_log4j_properties = config['configurations']['spark-log4j-properties']['content']
@@ -117,7 +103,7 @@ spark_javaopts_properties = config['configurations']['spark-javaopts-properties'
hive_server_host = default("/clusterHostInfo/hive_server_host", [])
is_hive_installed = not len(hive_server_host) == 0
-hdp_full_version = get_hdp_version()
+hdp_full_version = functions.get_hdp_version('spark-client')
spark_driver_extraJavaOptions = str(config['configurations']['spark-defaults']['spark.driver.extraJavaOptions'])
if spark_driver_extraJavaOptions.find('-Dhdp.version') == -1:
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
index 7b8ba38..f5b0167 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/service_check.py
@@ -27,40 +27,16 @@ from resource_management.core.logger import Logger
class SparkServiceCheck(Script):
def service_check(self, env):
import params
-
env.set_params(params)
if params.security_enabled:
spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
Execute(spark_kinit_cmd, user=params.spark_user)
- command = "curl"
- httpGssnegotiate = "--negotiate"
- userpswd = "-u:"
- insecure = "-k"
- silent = "-s"
- out = "-o /dev/null"
- head = "-w'%{http_code}'"
- url = 'http://' + params.spark_history_server_host + ':' + str(params.spark_history_ui_port)
-
- command_with_flags = [command, silent, out, head, httpGssnegotiate, userpswd, insecure, url]
-
- is_running = False
- for i in range(1,11):
- proc = subprocess.Popen(command_with_flags, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- Logger.info("Try %d, command: %s" % (i, " ".join(command_with_flags)))
- (stdout, stderr) = proc.communicate()
- response = stdout
- if '200' in response:
- is_running = True
- Logger.info('Spark Job History Server up and running')
- break
- Logger.info("Response: %s" % str(response))
- time.sleep(5)
-
- if is_running == False :
- Logger.info('Spark Job History Server not running.')
- raise ComponentIsNotRunning()
+ Execute(format("curl -s -o /dev/null -w'%{{http_code}}' --negotiate -u: -k http://{spark_history_server_host}:{spark_history_ui_port} | grep 200"),
+ tries = 10,
+ try_sleep=3,
+ )
if __name__ == "__main__":
SparkServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
index 6935b08..d8fbc8c 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
@@ -31,25 +31,22 @@ from resource_management.core import shell
def setup_spark(env, type, action = None):
import params
- env.set_params(params)
-
Directory([params.spark_pid_dir, params.spark_log_dir],
owner=params.spark_user,
group=params.user_group,
recursive=True
)
- if type == 'server':
- if action == 'start' or action == 'config':
- params.HdfsDirectory(params.spark_hdfs_user_dir,
- action="create",
- owner=params.spark_user,
- mode=0775
- )
-
- file_path = params.spark_conf + '/spark-defaults.conf'
- create_file(file_path)
-
- write_properties_to_file(file_path, spark_properties(params))
+ if type == 'server' and action == 'config':
+ params.HdfsDirectory(params.spark_hdfs_user_dir,
+ action="create",
+ owner=params.spark_user,
+ mode=0775
+ )
+
+ PropertiesFile(format("{spark_conf}/spark-defaults.conf"),
+ properties = params.config['configurations']['spark-defaults'],
+ key_value_delimiter = " ",
+ )
# create spark-env.sh in etc/conf dir
File(os.path.join(params.spark_conf, 'spark-env.sh'),
@@ -79,127 +76,9 @@ def setup_spark(env, type, action = None):
)
if params.is_hive_installed:
- hive_config = get_hive_config()
XmlConfig("hive-site.xml",
conf_dir=params.spark_conf,
- configurations=hive_config,
+ configurations=params.config['configurations']['hive-site'],
owner=params.spark_user,
group=params.spark_group,
mode=0644)
-
-def get_hive_config():
- import params
- hive_conf_dict = dict()
- hive_conf_dict['hive.metastore.uris'] = params.config['configurations']['hive-site']['hive.metastore.uris']
- if params.security_enabled:
- hive_conf_dict['hive.metastore.sasl.enabled'] = str(params.config['configurations']['hive-site']['hive.metastore.sasl.enabled']).lower()
- hive_conf_dict['hive.metastore.kerberos.keytab.file'] = params.config['configurations']['hive-site']['hive.metastore.kerberos.keytab.file']
- hive_conf_dict['hive.server2.authentication.spnego.principal'] = params.config['configurations']['hive-site']['hive.server2.authentication.spnego.principal']
- hive_conf_dict['hive.server2.authentication.spnego.keytab'] = params.config['configurations']['hive-site']['hive.server2.authentication.spnego.keytab']
- hive_conf_dict['hive.metastore.kerberos.principal'] = params.config['configurations']['hive-site']['hive.metastore.kerberos.principal']
- hive_conf_dict['hive.server2.authentication.kerberos.principal'] = params.config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal']
- hive_conf_dict['hive.server2.authentication.kerberos.keytab'] = params.config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab']
- hive_conf_dict['hive.security.authorization.enabled'] = params.spark_hive_sec_authorization_enabled
- hive_conf_dict['hive.server2.enable.doAs'] = str(params.config['configurations']['hive-site']['hive.server2.enable.doAs']).lower()
-
- return hive_conf_dict
-
-
-def spark_properties(params):
- spark_dict = dict()
-
- all_spark_config = params.config['configurations']['spark-defaults']
- #Add all configs unfiltered first to handle Custom case.
- spark_dict = all_spark_config.copy()
-
- spark_dict['spark.yarn.executor.memoryOverhead'] = params.spark_yarn_executor_memoryOverhead
- spark_dict['spark.yarn.driver.memoryOverhead'] = params.spark_yarn_driver_memoryOverhead
- spark_dict['spark.yarn.applicationMaster.waitTries'] = params.spark_yarn_applicationMaster_waitTries
- spark_dict['spark.yarn.scheduler.heartbeat.interval-ms'] = params.spark_yarn_scheduler_heartbeat_interval
- spark_dict['spark.yarn.max_executor.failures'] = params.spark_yarn_max_executor_failures
- spark_dict['spark.yarn.queue'] = params.spark_yarn_queue
- spark_dict['spark.yarn.containerLauncherMaxThreads'] = params.spark_yarn_containerLauncherMaxThreads
- spark_dict['spark.yarn.submit.file.replication'] = params.spark_yarn_submit_file_replication
- spark_dict['spark.yarn.preserve.staging.files'] = params.spark_yarn_preserve_staging_files
-
- # Hardcoded paramaters to be added to spark-defaults.conf
- spark_dict['spark.yarn.historyServer.address'] = params.spark_history_server_host + ':' + str(
- params.spark_history_ui_port)
- spark_dict['spark.yarn.services'] = 'org.apache.spark.deploy.yarn.history.YarnHistoryService'
- spark_dict['spark.history.provider'] = 'org.apache.spark.deploy.yarn.history.YarnHistoryProvider'
- spark_dict['spark.history.ui.port'] = params.spark_history_ui_port
-
- spark_dict['spark.driver.extraJavaOptions'] = params.spark_driver_extraJavaOptions
- spark_dict['spark.yarn.am.extraJavaOptions'] = params.spark_yarn_am_extraJavaOptions
-
-
- return spark_dict
-
-
-def write_properties_to_file(file_path, value):
- for key in value:
- modify_config(file_path, key, value[key])
-
-
-def modify_config(filepath, variable, setting):
- var_found = False
- already_set = False
- V = str(variable)
- S = str(setting)
-
- if ' ' in S:
- S = '%s' % S
-
- for line in fileinput.input(filepath, inplace=1):
- if not line.lstrip(' ').startswith('#') and '=' in line:
- _infile_var = str(line.split('=')[0].rstrip(' '))
- _infile_set = str(line.split('=')[1].lstrip(' ').rstrip())
- if var_found == False and _infile_var.rstrip(' ') == V:
- var_found = True
- if _infile_set.lstrip(' ') == S:
- already_set = True
- else:
- line = "%s %s\n" % (V, S)
-
- sys.stdout.write(line)
-
- if not var_found:
- with open(filepath, "a") as f:
- f.write("%s \t %s\n" % (V, S))
- elif already_set == True:
- pass
- else:
- pass
-
- return
-
-
-def create_file(file_path):
- try:
- file = open(file_path, 'w')
- file.close()
- except:
- print('Unable to create file: ' + file_path)
- sys.exit(0)
-
-
-def get_hdp_version():
- try:
- command = 'hdp-select status hadoop-client'
- return_code, hdp_output = shell.call(command, timeout=20)
- except Exception, e:
- Logger.error(str(e))
- raise Fail('Unable to execute hdp-select command to retrieve the version.')
-
- if return_code != 0:
- raise Fail(
- 'Unable to determine the current version because of a non-zero return code of {0}'.format(str(return_code)))
-
- hdp_version = re.sub('hadoop-client - ', '', hdp_output)
- hdp_version = hdp_version.rstrip()
- match = re.match('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', hdp_version)
-
- if match is None:
- raise Fail('Failed to get extracted version')
-
- return hdp_version
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
index 970dc6d..9f9e3dc 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_client.py
@@ -28,29 +28,28 @@ from setup_spark import setup_spark
class SparkClient(Script):
- def get_stack_to_component(self):
- return {"HDP": "spark-client"}
-
- def pre_rolling_restart(self, env):
- import params
-
- env.set_params(params)
- if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
- Execute(format("hdp-select set spark-client {version}"))
-
def install(self, env):
self.install_packages(env)
self.configure(env)
def configure(self, env):
import params
-
env.set_params(params)
+
setup_spark(env, 'client', action = 'config')
def status(self, env):
raise ClientComponentHasNoStatus()
+
+ def get_stack_to_component(self):
+ return {"HDP": "spark-client"}
+ def pre_rolling_restart(self, env):
+ import params
+
+ env.set_params(params)
+ if params.version and compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') >= 0:
+ Execute(format("hdp-select set spark-client {version}"))
if __name__ == "__main__":
SparkClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
new file mode 100644
index 0000000..26a403d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/spark_service.py
@@ -0,0 +1,46 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from resource_management import *
+
+def spark_service(action):
+ import params
+
+ if action == 'start':
+ if params.security_enabled:
+ spark_kinit_cmd = format("{kinit_path_local} -kt {spark_kerberos_keytab} {spark_principal}; ")
+ Execute(spark_kinit_cmd, user=params.spark_user)
+
+ copy_tarballs_to_hdfs('tez', 'spark-historyserver', params.spark_user, params.hdfs_user, params.user_group)
+
+ no_op_test = format(
+ 'ls {spark_history_server_pid_file} >/dev/null 2>&1 && ps -p `cat {spark_history_server_pid_file}` >/dev/null 2>&1')
+ Execute(format('{spark_history_server_start}'),
+ user=params.spark_user,
+ environment={'JAVA_HOME': params.java_home},
+ not_if=no_op_test
+ )
+ elif action == 'stop':
+ Execute(format('{spark_history_server_stop}'),
+ user=params.spark_user,
+ environment={'JAVA_HOME': params.java_home}
+ )
+ File(params.spark_history_server_pid_file,
+ action="delete"
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index c5ebbc0..8ffbd67 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -83,29 +83,28 @@ class TestKnoxGateway(RMFTestCase):
'/etc/knox/conf'),
sudo = True,
)
- self.assertResourceCalled('Execute', '/usr/lib/knox/bin/knoxcli.sh create-master --master sa',
+ self.assertResourceCalled('Execute', '/usr/hdp/current/knox-server/bin/knoxcli.sh create-master --master sa',
environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
not_if = "ambari-sudo.sh su knox -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]test -f /var/lib/knox/data/security/master'",
user = 'knox',
)
- self.assertResourceCalled('Execute', '/usr/lib/knox/bin/knoxcli.sh create-cert --hostname c6401.ambari.apache.org',
+ self.assertResourceCalled('Execute', '/usr/hdp/current/knox-server/bin/knoxcli.sh create-cert --hostname c6401.ambari.apache.org',
environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
not_if = "ambari-sudo.sh su knox -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]test -f /var/lib/knox/data/security/keystores/gateway.jks'",
user = 'knox',
)
self.assertResourceCalled('File', '/etc/knox/conf/ldap-log4j.properties',
- mode=0644,
- group='knox',
- owner = 'knox',
- content = self.getConfig()['configurations']['ldap-log4j']['content']
+ content = '\n # Licensed to the Apache Software Foundation (ASF) under one\n # or more contributor license agreements. See the NOTICE file\n # distributed with this work for additional information\n # regarding copyright ownership. The ASF licenses this file\n # to you under the Apache License, Version 2.0 (the\n # "License"); you may not use this file except in compliance\n # with the License. You may obtain a copy of the License at\n #\n # http://www.apache.org/licenses/LICENSE-2.0\n #\n # Unless required by applicable law or agreed to in writing, software\n # distributed under the License is distributed on an "AS IS" BASIS,\n # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n # See the License for the specific language governing permissions and\n # limitations under the License.\n #testing\n\n app.log.dir=${launcher.dir}/.
./logs\n app.log.file=${launcher.name}.log\n\n log4j.rootLogger=ERROR, drfa\n log4j.logger.org.apache.directory.server.ldap.LdapServer=INFO\n log4j.logger.org.apache.directory=WARN\n\n log4j.appender.stdout=org.apache.log4j.ConsoleAppender\n log4j.appender.stdout.layout=org.apache.log4j.PatternLayout\n log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n\n\n log4j.appender.drfa=org.apache.log4j.DailyRollingFileAppender\n log4j.appender.drfa.File=${app.log.dir}/${app.log.file}\n log4j.appender.drfa.DatePattern=.yyyy-MM-dd\n log4j.appender.drfa.layout=org.apache.log4j.PatternLayout\n log4j.appender.drfa.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n',
+ owner = 'knox',
+ group = 'knox',
+ mode = 0644,
)
self.assertResourceCalled('File', '/etc/knox/conf/users.ldif',
- mode=0644,
- group='knox',
- owner = 'knox',
- content = self.getConfig()['configurations']['users-ldif']['content']
+ content = '\n # Licensed to the Apache Software Foundation (ASF) under one\n # or more contributor license agreements. See the NOTICE file\n # distributed with this work for additional information\n # regarding copyright ownership. The ASF licenses this file\n # to you under the Apache License, Version 2.0 (the\n # "License"); you may not use this file except in compliance\n # with the License. You may obtain a copy of the License at\n #\n # http://www.apache.org/licenses/LICENSE-2.0\n #\n # Unless required by applicable law or agreed to in writing, software\n # distributed under the License is distributed on an "AS IS" BASIS,\n # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n # See the License for the specific language governing permissions and\n # limitations under the License
.\n\n version: 1\n\n # Please replace with site specific values\n dn: dc=hadoop,dc=apache,dc=org\n objectclass: organization\n objectclass: dcObject\n o: Hadoop\n dc: hadoop\n\n # Entry for a sample people container\n # Please replace with site specific values\n dn: ou=people,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass:organizationalUnit\n ou: people\n\n # Entry for a sample end user\n # Please replace with site specific values\n dn: uid=guest,ou=people,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass:person\n objectclass:organizationalPerson\n objectclass:inetOrgPerson\n cn: Guest\n sn: User\n uid: guest\n userPassword:guest-password\n\n # entry for sample user admin\n dn: uid
=admin,ou=people,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass:person\n objectclass:organizationalPerson\n objectclass:inetOrgPerson\n cn: Admin\n sn: Admin\n uid: admin\n userPassword:admin-password\n\n # entry for sample user sam\n dn: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass:person\n objectclass:organizationalPerson\n objectclass:inetOrgPerson\n cn: sam\n sn: sam\n uid: sam\n userPassword:sam-password\n\n # entry for sample user tom\n dn: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass:person\n objectclass:organizationalPerson\n objectclass:inetOrgPerson\n cn: tom\n sn: tom\n uid: tom\n userPassword:t
om-password\n\n # create FIRST Level groups branch\n dn: ou=groups,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass:organizationalUnit\n ou: groups\n description: generic groups branch\n\n # create the analyst group under groups\n dn: cn=analyst,ou=groups,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass: groupofnames\n cn: analyst\n description:analyst group\n member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org\n member: uid=tom,ou=people,dc=hadoop,dc=apache,dc=org\n\n\n # create the scientist group under groups\n dn: cn=scientist,ou=groups,dc=hadoop,dc=apache,dc=org\n objectclass:top\n objectclass: groupofnames\n cn: scientist\n description: scientist group\n member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org',
+ owner = 'knox',
+ group = 'knox',
+ mode = 0644,
)
-
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py b/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
index 9ad4b02..2a03cde 100644
--- a/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
+++ b/ambari-server/src/test/python/stacks/2.2/SLIDER/test_slider_client.py
@@ -79,12 +79,11 @@ class TestSliderClient(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Execute',
- '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa@EXAMPLE.COM; /usr/lib/slider/bin/slider list',
- logoutput=True,
- tries=3,
- user='ambari-qa',
- try_sleep=5,
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/smokeuser.headless.keytab ambari-qa@EXAMPLE.COM; /usr/hdp/current/slider-client/bin/slider list',
+ logoutput = True,
+ tries = 3,
+ user = 'ambari-qa',
+ try_sleep = 5,
)
self.assertNoMoreResources()
@@ -98,11 +97,11 @@ class TestSliderClient(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- self.assertResourceCalled('Execute', ' /usr/lib/slider/bin/slider list',
- logoutput=True,
- tries=3,
- user='ambari-qa',
- try_sleep=5,
+ self.assertResourceCalled('Execute', ' /usr/hdp/current/slider-client/bin/slider list',
+ logoutput = True,
+ tries = 3,
+ user = 'ambari-qa',
+ try_sleep = 5,
)
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
index 469f68e..6461964 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
@@ -17,90 +17,195 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
'''
-import sys
-import os
from mock.mock import MagicMock, patch
-
from stacks.utils.RMFTestCase import *
-from resource_management.core import shell
-from resource_management.libraries.functions import dynamic_variable_interpretation
+@patch("resource_management.libraries.functions.get_hdp_version", new=MagicMock(return_value="2.3.0.0-1597"))
class TestJobHistoryServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
STACK_VERSION = "2.2"
- def setUp(self):
- sys.path.insert(0, os.path.join(os.getcwd(),
- "../../main/resources/common-services", self.COMMON_SERVICES_PACKAGE_DIR,
- "scripts"))
-
- @patch.object(shell, "call")
- @patch("setup_spark.create_file")
- @patch("setup_spark.write_properties_to_file")
- @patch.object(dynamic_variable_interpretation, "copy_tarballs_to_hdfs")
- def test_start(self, copy_tarball_mock, write_properties_to_file_mock, create_file_mock, call_mock):
- hdp_version = "2.2.2.0-2538"
- call_mock.return_value = (0, hdp_version)
- copy_tarball_mock.return_value = 0
-
+ def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
- classname="JobHistoryServer",
- command="start",
- config_file="spark-job-history-server.json",
- hdp_stack_version=self.STACK_VERSION,
- target=RMFTestCase.TARGET_COMMON_SERVICES
+ classname = "JobHistoryServer",
+ command = "configure",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
-
- self.assertTrue(create_file_mock.called)
- self.assertTrue(write_properties_to_file_mock.called)
-
-
- self.assertResourceCalled("Directory", "/var/run/spark",
- owner="spark",
- group="hadoop",
- recursive=True
- )
- self.assertResourceCalled("Directory", "/var/log/spark",
- owner="spark",
- group="hadoop",
- recursive=True
- )
- self.assertResourceCalled("HdfsDirectory", "/user/spark",
- security_enabled=False,
- keytab=UnknownConfigurationMock(),
- conf_dir="/etc/hadoop/conf",
- hdfs_user="hdfs",
- kinit_path_local="/usr/bin/kinit",
- mode=509,
- owner="spark",
- bin_dir="/usr/hdp/current/hadoop-client/bin",
- action=["create"]
- )
- self.assertResourceCalled("File", "/etc/spark/conf/spark-env.sh",
- owner="spark",
- group="spark",
- content=InlineTemplate(self.getConfig()['configurations']['spark-env']['content'])
- )
- self.assertResourceCalled("File", "/etc/spark/conf/log4j.properties",
- owner="spark",
- group="spark",
- content=self.getConfig()['configurations']['spark-log4j-properties']['content']
- )
- self.assertResourceCalled("File", "/etc/spark/conf/metrics.properties",
- owner="spark",
- group="spark",
- content=InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content'])
- )
- self.assertResourceCalled("File", "/etc/spark/conf/java-opts",
- owner="spark",
- group="spark",
- content=" -Dhdp.version=" + hdp_version
+ self.assert_configure_default()
+ self.assertNoMoreResources()
+
+ def test_start_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
+ classname = "JobHistoryServer",
+ command = "start",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
-
- copy_tarball_mock.assert_called_with("tez", "spark-historyserver", "spark", "hdfs", "hadoop")
+ self.assert_configure_default()
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-history-server.sh',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1',
+ user = 'spark',
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
+ classname = "JobHistoryServer",
+ command = "stop",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/stop-history-server.sh',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ user = 'spark',
+ )
+ self.assertResourceCalled('File', '/var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
- self.assertResourceCalled("Execute", "/usr/hdp/current/spark-historyserver/sbin/start-history-server.sh",
- not_if="ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1",
- environment={'JAVA_HOME': '/usr/jdk64/jdk1.7.0_67'},
- user="spark"
+ def test_configure_secured(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
+ classname = "JobHistoryServer",
+ command = "configure",
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured()
+ self.assertNoMoreResources()
+
+ def test_start_secured(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
+ classname = "JobHistoryServer",
+ command = "start",
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured()
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/spark.service.keytab spark/localhost@EXAMPLE.COM; ',
+ user = 'spark',
+ )
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/start-history-server.sh',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ not_if = 'ls /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid >/dev/null 2>&1 && ps -p `cat /var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid` >/dev/null 2>&1',
+ user = 'spark',
+ )
+ self.assertNoMoreResources()
+
+ def test_stop_secured(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/job_history_server.py",
+ classname = "JobHistoryServer",
+ command = "stop",
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', '/usr/hdp/current/spark-client/sbin/stop-history-server.sh',
+ environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
+ user = 'spark',
+ )
+ self.assertResourceCalled('File', '/var/run/spark/spark-spark-org.apache.spark.deploy.history.HistoryServer-1.pid',
+ action = ['delete'],
+ )
+ self.assertNoMoreResources()
+
+ def assert_configure_default(self):
+ self.assertResourceCalled('Directory', '/var/run/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/var/log/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('HdfsDirectory', '/user/spark',
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ kinit_path_local = '/usr/bin/kinit',
+ mode = 0775,
+ owner = 'spark',
+ bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ action = ['create'],
+ )
+ self.assertResourceCalled('PropertiesFile', '/etc/spark/conf/spark-defaults.conf',
+ key_value_delimiter = ' ',
+ properties = self.getConfig()['configurations']['spark-defaults'],
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/spark-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-env']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/log4j.properties',
+ content = '\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO',
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/metrics.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/java-opts',
+ content = ' -Dhdp.version=2.3.0.0-1597',
+ owner = 'spark',
+ group = 'spark',
+ )
+
+ def assert_configure_secured(self):
+ self.assertResourceCalled('Directory', '/var/run/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/var/log/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('HdfsDirectory', '/user/spark',
+ security_enabled = True,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = UnknownConfigurationMock(),
+ kinit_path_local = '/usr/bin/kinit',
+ mode = 0775,
+ owner = 'spark',
+ bin_dir = '/usr/hdp/current/hadoop-client/bin',
+ action = ['create'],
+ )
+ self.assertResourceCalled('PropertiesFile', '/etc/spark/conf/spark-defaults.conf',
+ key_value_delimiter = ' ',
+ properties = self.getConfig()['configurations']['spark-defaults'],
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/spark-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-env']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/log4j.properties',
+ content = '\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO',
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/metrics.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/java-opts',
+ content = ' -Dhdp.version=2.3.0.0-1597',
+ owner = 'spark',
+ group = 'spark',
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
new file mode 100644
index 0000000..30235f0
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
@@ -0,0 +1,120 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from mock.mock import MagicMock, patch
+from stacks.utils.RMFTestCase import *
+
+@patch("resource_management.libraries.functions.get_hdp_version", new=MagicMock(return_value="2.3.0.0-1597"))
+class TestSparkClient(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
+ STACK_VERSION = "2.2"
+
+ def test_configure_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_client.py",
+ classname = "SparkClient",
+ command = "configure",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_default()
+ self.assertNoMoreResources()
+
+ def test_configure_secured(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/spark_client.py",
+ classname = "SparkClient",
+ command = "configure",
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assert_configure_secured()
+ self.assertNoMoreResources()
+
+ def assert_configure_default(self):
+ self.assertResourceCalled('Directory', '/var/run/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/var/log/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('PropertiesFile', '/etc/spark/conf/spark-defaults.conf',
+ key_value_delimiter = ' ',
+ properties = self.getConfig()['configurations']['spark-defaults'],
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/spark-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-env']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/log4j.properties',
+ content = '\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO',
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/metrics.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/java-opts',
+ content = ' -Dhdp.version=2.3.0.0-1597',
+ owner = 'spark',
+ group = 'spark',
+ )
+
+ def assert_configure_secured(self):
+ self.assertResourceCalled('Directory', '/var/run/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('Directory', '/var/log/spark',
+ owner = 'spark',
+ group = 'hadoop',
+ recursive = True,
+ )
+ self.assertResourceCalled('PropertiesFile', '/etc/spark/conf/spark-defaults.conf',
+ key_value_delimiter = ' ',
+ properties = self.getConfig()['configurations']['spark-defaults'],
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/spark-env.sh',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-env']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/log4j.properties',
+ content = '\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO',
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/metrics.properties',
+ content = InlineTemplate(self.getConfig()['configurations']['spark-metrics-properties']['content']),
+ owner = 'spark',
+ group = 'spark',
+ )
+ self.assertResourceCalled('File', '/etc/spark/conf/java-opts',
+ content = ' -Dhdp.version=2.3.0.0-1597',
+ owner = 'spark',
+ group = 'spark',
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_service_check.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_service_check.py
new file mode 100644
index 0000000..a02a908
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_service_check.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from mock.mock import MagicMock, call, patch
+from stacks.utils.RMFTestCase import *
+
+@patch("resource_management.libraries.functions.get_hdp_version", new=MagicMock(return_value="2.3.0.0-1597"))
+class TestServiceCheck(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "SPARK/1.2.0.2.2/package"
+ STACK_VERSION = "2.2"
+
+ def test_service_check_default(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
+ classname="SparkServiceCheck",
+ command="service_check",
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', "curl -s -o /dev/null -w'%{http_code}' --negotiate -u: -k http://localhost:18080 | grep 200",
+ tries = 10,
+ try_sleep = 3,
+ )
+ self.assertNoMoreResources()
+
+
+ def test_service_check_secured(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/service_check.py",
+ classname="SparkServiceCheck",
+ command="service_check",
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
+ )
+ self.assertResourceCalled('Execute', '/usr/bin/kinit -kt /etc/security/keytabs/spark.service.keytab spark/localhost@EXAMPLE.COM; ',
+ user = 'spark',
+ )
+ self.assertResourceCalled('Execute', "curl -s -o /dev/null -w'%{http_code}' --negotiate -u: -k http://localhost:18080 | grep 200",
+ tries = 10,
+ try_sleep = 3,
+ )
+ self.assertNoMoreResources()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/85acd9fa/ambari-server/src/test/python/stacks/2.2/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/default.json b/ambari-server/src/test/python/stacks/2.2/configs/default.json
index 0a9a132..892cdd3 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/default.json
@@ -7,7 +7,7 @@
"ambari_db_rca_password": "mapred",
"ambari_db_rca_url": "jdbc:postgresql://c6401.ambari.apache.org/ambarirca",
"jce_name": "UnlimitedJCEPolicyJDK7.zip",
- "stack_version": "2.1",
+ "stack_version": "2.2",
"stack_name": "HDP",
"ambari_db_rca_driver": "org.postgresql.Driver",
"jdk_name": "jdk-7u67-linux-x64.tar.gz",
@@ -31,6 +31,58 @@
"taskId": 152,
"public_hostname": "c6401.ambari.apache.org",
"configurations": {
+ "spark-defaults": {
+ "spark.yarn.applicationMaster.waitTries": "10",
+ "spark.history.kerberos.keytab": "none",
+ "spark.yarn.preserve.staging.files": "false",
+ "spark.yarn.submit.file.replication": "3",
+ "spark.history.kerberos.principal": "none",
+ "spark.yarn.driver.memoryOverhead": "384",
+ "spark.yarn.queue": "default",
+ "spark.yarn.containerLauncherMaxThreads": "25",
+ "spark.yarn.scheduler.heartbeat.interval-ms": "5000",
+ "spark.history.ui.port": "18080",
+ "spark.yarn.max.executor.failures": "3",
+ "spark.driver.extraJavaOptions": "",
+ "spark.history.provider": "org.apache.spark.deploy.yarn.history.YarnHistoryProvider",
+ "spark.yarn.am.extraJavaOptions": "",
+ "spark.yarn.executor.memoryOverhead": "384"
+ },
+ "spark-javaopts-properties": {
+ "content": " "
+ },
+ "spark-log4j-properties": {
+ "content": "\n# Set everything to be logged to the console\nlog4j.rootCategory=INFO, console\nlog4j.appender.console=org.apache.log4j.ConsoleAppender\nlog4j.appender.console.target=System.err\nlog4j.appender.console.layout=org.apache.log4j.PatternLayout\nlog4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n\n\n# Settings to quiet third party logs that are too verbose\nlog4j.logger.org.eclipse.jetty=WARN\nlog4j.logger.org.eclipse.jetty.util.component.AbstractLifeCycle=ERROR\nlog4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO\nlog4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO"
+ },
+ "spark-env": {
+ "content": "\n#!/usr/bin/env bash\n\n# This file is sourced when running various Spark programs.\n# Copy it as spark-env.sh and edit that to configure Spark for your site.\n\n# Options read in YARN client mode\n#SPARK_EXECUTOR_INSTANCES=\"2\" #Number of workers to start (Default: 2)\n#SPARK_EXECUTOR_CORES=\"1\" #Number of cores for the workers (Default: 1).\n#SPARK_EXECUTOR_MEMORY=\"1G\" #Memory per Worker (e.g. 1000M, 2G) (Default: 1G)\n#SPARK_DRIVER_MEMORY=\"512 Mb\" #Memory for Master (e.g. 1000M, 2G) (Default: 512 Mb)\n#SPARK_YARN_APP_NAME=\"spark\" #The name of your application (Default: Spark)\n#SPARK_YARN_QUEUE=\"~@~Xdefault~@~Y\" #The hadoop queue to use for allocation requests (Default: @~Xdefault~@~Y)\n#SPARK_YARN_DIST_FILES=\"\" #Comma separated list of files to be distributed with the job.\n#SPARK_YARN_DIST_ARCHIVES=\"\" #Comma separated list of archives to be distributed with the job.\n\n# Generic options for the daemons used in the standalone deploy mode\n\
n# Alternate conf dir. (Default: ${SPARK_HOME}/conf)\nexport SPARK_CONF_DIR=${SPARK_HOME:-{{spark_home}}}/conf\n\n# Where log files are stored.(Default:${SPARK_HOME}/logs)\n#export SPARK_LOG_DIR=${SPARK_HOME:-{{spark_home}}}/logs\nexport SPARK_LOG_DIR={{spark_log_dir}}\n\n# Where the pid file is stored. (Default: /tmp)\nexport SPARK_PID_DIR={{spark_pid_dir}}\n\n# A string representing this instance of spark.(Default: $USER)\nSPARK_IDENT_STRING=$USER\n\n# The scheduling priority for daemons. (Default: 0)\nSPARK_NICENESS=0\n\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\nexport HADOOP_CONF_DIR=${HADOOP_CONF_DIR:-{{hadoop_conf_dir}}}\n\n# The java implementation to use.\nexport JAVA_HOME={{java_home}}\n\nif [ -d \"/etc/tez/conf/\" ]; then\n export TEZ_CONF_DIR=/etc/tez/conf\nelse\n export TEZ_CONF_DIR=\nfi",
+ "spark_pid_dir": "/var/run/spark",
+ "spark_log_dir": "/var/log/spark",
+ "spark_group": "spark",
+ "spark_user": "spark"
+ },
+ "spark-metrics-properties": {
+ "content": "\n# syntax: [instance].sink|source.[name].[options]=[value]\n\n# This file configures Spark's internal metrics system. The metrics system is\n# divided into instances which correspond to internal components.\n# Each instance can be configured to report its metrics to one or more sinks.\n# Accepted values for [instance] are \"master\", \"worker\", \"executor\", \"driver\",\n# and \"applications\". A wild card \"*\" can be used as an instance name, in\n# which case all instances will inherit the supplied property.\n#\n# Within an instance, a \"source\" specifies a particular set of grouped metrics.\n# there are two kinds of sources:\n# 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will\n# collect a Spark component's internal state. Each instance is paired with a\n# Spark source that is added automatically.\n# 2. Common sources, like JvmSource, which will collect low level state.\n# These can be added through configuration options and ar
e then loaded\n# using reflection.\n#\n# A \"sink\" specifies where metrics are delivered to. Each instance can be\n# assigned one or more sinks.\n#\n# The sink|source field specifies whether the property relates to a sink or\n# source.\n#\n# The [name] field specifies the name of source or sink.\n#\n# The [options] field is the specific property of this source or sink. The\n# source or sink is responsible for parsing this property.\n#\n# Notes:\n# 1. To add a new sink, set the \"class\" option to a fully qualified class\n# name (see examples below).\n# 2. Some sinks involve a polling period. The minimum allowed polling period\n# is 1 second.\n# 3. Wild card properties can be overridden by more specific properties.\n# For example, master.sink.console.period takes precedence over\n# *.sink.console.period.\n# 4. A metrics specific configuration\n# \"spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties\" should be\n# added to Java properties using -Dspark.metrics.conf=xxx if you wa
nt to\n# customize metrics system. You can also put the file in ${SPARK_HOME}/conf\n# and it will be loaded automatically.\n# 5. MetricsServlet is added by default as a sink in master, worker and client\n# driver, you can send http request \"/metrics/json\" to get a snapshot of all the\n# registered metrics in json format. For master, requests \"/metrics/master/json\" and\n# \"/metrics/applications/json\" can be sent seperately to get metrics snapshot of\n# instance master and applications. MetricsServlet may not be configured by self.\n#\n\n## List of available sinks and their properties.\n\n# org.apache.spark.metrics.sink.ConsoleSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n\n# org.apache.spark.metrics.sink.CSVSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n# directory /tmp Where to store CSV files\n\n# org.apache.spark.metrics.sink.GangliaSink\n# Name: Default: Description:\n#
host NONE Hostname or multicast group of Ganglia server\n# port NONE Port of Ganglia server(s)\n# period 10 Poll period\n# unit seconds Units of poll period\n# ttl 1 TTL of messages sent by Ganglia\n# mode multicast Ganglia network mode ('unicast' or 'multicast')\n\n# org.apache.spark.metrics.sink.JmxSink\n\n# org.apache.spark.metrics.sink.MetricsServlet\n# Name: Default: Description:\n# path VARIES* Path prefix from the web server root\n# sample false Whether to show entire set of samples for histograms ('false' or 'true')\n#\n# * Default path is /metrics/json for all instances except the master. The master has two paths:\n# /metrics/aplications/json # App information\n# /metrics/master/json # Master information\n\n# org.apache.spark.metrics.sink.GraphiteSink\n# Name: Default: Description:\n# host NONE Hostname of Graphite server\n# port NONE Port of Graphite server\n# period 10 Poll period\n# unit seconds Units of poll period\n# prefix EMPTY STRING Prefix to prepend to metric name
\n\n## Examples\n# Enable JmxSink for all instances by class name\n#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink\n\n# Enable ConsoleSink for all instances by class name\n#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink\n\n# Polling period for ConsoleSink\n#*.sink.console.period=10\n\n#*.sink.console.unit=seconds\n\n# Master instance overlap polling period\n#master.sink.console.period=15\n\n#master.sink.console.unit=seconds\n\n# Enable CsvSink for all instances\n#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink\n\n# Polling period for CsvSink\n#*.sink.csv.period=1\n\n#*.sink.csv.unit=minutes\n\n# Polling directory for CsvSink\n#*.sink.csv.directory=/tmp/\n\n# Worker instance overlap polling period\n#worker.sink.csv.period=10\n\n#worker.sink.csv.unit=minutes\n\n# Enable jvm source for instance master, worker, driver and executor\n#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#worker.source.jvm.class=org.apache.spark.metrics.source
.JvmSource\n\n#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource"
+ },
+ "hadoop-env": {
+ "dtnode_heapsize": "1024m",
+ "namenode_opt_maxnewsize": "256m",
+ "hdfs_log_dir_prefix": "/var/log/hadoop",
+ "namenode_heapsize": "1024m",
+ "proxyuser_group": "users",
+ "hadoop_pid_dir_prefix": "/var/run/hadoop",
+ "content": "\n# Set Hadoop-specific environment variables here.\n\n# The only required environment variable is JAVA_HOME. All others are\n# optional. When running a distributed configuration it is best to\n# set JAVA_HOME in this file, so that it is correctly defined on\n# remote nodes.\n\n# The java implementation to use. Required.\nexport JAVA_HOME={{java_home}}\nexport HADOOP_HOME_WARN_SUPPRESS=1\n\n# Hadoop home directory\nexport HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}\n\n# Hadoop Configuration Directory\n\n{# this is different for HDP1 #}\n# Path to jsvc required by secure HDP 2.0 datanode\nexport JSVC_HOME={{jsvc_path}}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\nexport HADOOP_HEAPSIZE=\"{{hadoop_heapsize}}\"\n\nexport HADOOP_NAMENODE_INIT_HEAPSIZE=\"-Xms{{namenode_heapsize}}\"\n\n# Extra Java runtime options. Empty by default.\nexport HADOOP_OPTS=\"-Djava.net.preferIPv4Stack=true ${HADOOP_OPTS}\"\n\n# Command specific options appende
d to HADOOP_OPTS when specified\nexport HADOOP_NAMENODE_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{namenode_opt_newsize}} -XX:MaxNewSize={{namenode_opt_maxnewsize}} -XX:PermSize={{namenode_opt_permsize}} -XX:MaxPermSize={{namenode_opt_maxpermsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{namenode_heapsize}} -Xmx{{namenode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_NAMENODE_OPTS}\"\nHADOOP_JOBTRACKER_OPTS=\"-server -XX:ParallelGCThreads=8 -XX:+UseConcMarkSweepGC -XX:ErrorFile={{hdfs_log_dir_prefix}}/$USER/hs_err_pid%p.log -XX:NewSize={{jtnode_opt_newsize}} -XX:MaxNewSize={{jtnode_opt_maxnewsize}} -Xloggc:{{hdfs_log_dir_prefix}}/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStam
ps -Xmx{{jtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dmapred.audit.logger=INFO,MRAUDIT -Dhadoop.mapreduce.jobsummary.logger=INFO,JSA ${HADOOP_JOBTRACKER_OPTS}\"\n\nHADOOP_TASKTRACKER_OPTS=\"-server -Xmx{{ttnode_heapsize}} -Dhadoop.security.logger=ERROR,console -Dmapred.audit.logger=ERROR,console ${HADOOP_TASKTRACKER_OPTS}\"\nexport HADOOP_DATANODE_OPTS=\"-server -XX:ParallelGCThreads=4 -XX:+UseConcMarkSweepGC -XX:ErrorFile=/var/log/hadoop/$USER/hs_err_pid%p.log -XX:NewSize=200m -XX:MaxNewSize=200m -XX:PermSize=128m -XX:MaxPermSize=256m -Xloggc:/var/log/hadoop/$USER/gc.log-`date +'%Y%m%d%H%M'` -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xms{{dtnode_heapsize}} -Xmx{{dtnode_heapsize}} -Dhadoop.security.logger=INFO,DRFAS -Dhdfs.audit.logger=INFO,DRFAAUDIT ${HADOOP_DATANODE_OPTS}\"\nHADOOP_BALANCER_OPTS=\"-server -Xmx{{hadoop_heapsize}}m ${HADOOP_BALANCER_OPTS}\"\n\nexport HADOOP_SECONDARYNAMENODE_OPTS=$HADOOP_NAMENODE_OPTS\n\n# The following
applies to multiple commands (fs, dfs, fsck, distcp etc)\nexport HADOOP_CLIENT_OPTS=\"-Xmx${HADOOP_HEAPSIZE}m -XX:MaxPermSize=512m $HADOOP_CLIENT_OPTS\"\n\n# On secure datanodes, user to run the datanode as after dropping privileges\nexport HADOOP_SECURE_DN_USER=${HADOOP_SECURE_DN_USER:-{{hadoop_secure_dn_user}}}\n\n# Extra ssh options. Empty by default.\nexport HADOOP_SSH_OPTS=\"-o ConnectTimeout=5 -o SendEnv=HADOOP_CONF_DIR\"\n\n# Where log files are stored. $HADOOP_HOME/logs by default.\nexport HADOOP_LOG_DIR={{hdfs_log_dir_prefix}}/$USER\n\n# History server logs\nexport HADOOP_MAPRED_LOG_DIR={{mapred_log_dir_prefix}}/$USER\n\n# Where log files are stored in the secure data environment.\nexport HADOOP_SECURE_DN_LOG_DIR={{hdfs_log_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# File naming remote slave hosts. $HADOOP_HOME/conf/slaves by default.\n# export HADOOP_SLAVES=${HADOOP_HOME}/conf/slaves\n\n# host:path where hadoop code should be rsync'd from. Unset by default.\n# export HAD
OOP_MASTER=master:/home/$USER/src/hadoop\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HADOOP_SLAVE_SLEEP=0.1\n\n# The directory where pid files are stored. /tmp by default.\nexport HADOOP_PID_DIR={{hadoop_pid_dir_prefix}}/$USER\nexport HADOOP_SECURE_DN_PID_DIR={{hadoop_pid_dir_prefix}}/$HADOOP_SECURE_DN_USER\n\n# History server pid\nexport HADOOP_MAPRED_PID_DIR={{mapred_pid_dir_prefix}}/$USER\n\nYARN_RESOURCEMANAGER_OPTS=\"-Dyarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY\"\n\n# A string representing this instance of hadoop. $USER by default.\nexport HADOOP_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n\n# export HADOOP_NICENESS=10\n\n# Use libraries from standard classpath\nJAVA_JDBC_LIBS=\"\"\n#Add libraries required by mysql connector\nfor jarFile in `ls /usr/share/
java/*mysql* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by oracle connector\nfor jarFile in `ls /usr/share/java/*ojdbc* 2>/dev/null`\ndo\n JAVA_JDBC_LIBS=${JAVA_JDBC_LIBS}:$jarFile\ndone\n# Add libraries required by nodemanager\nMAPREDUCE_LIBS={{mapreduce_libs_path}}\nexport HADOOP_CLASSPATH=${HADOOP_CLASSPATH}${JAVA_JDBC_LIBS}:${MAPREDUCE_LIBS}\n\n# added to the HADOOP_CLASSPATH\nif [ -d \"/usr/hdp/current/tez-client\" ]; then\n if [ -d \"/etc/tez/conf/\" ]; then\n # When using versioned RPMs, the tez-client will be a symlink to the current folder of tez in HDP.\n export HADOOP_CLASSPATH=${HADOOP_CLASSPATH}:/usr/hdp/current/tez-client/*:/usr/hdp/current/tez-client/lib/*:/etc/tez/conf/\n fi\nfi\n\n\n# Setting path to hdfs command line\nexport HADOOP_LIBEXEC_DIR={{hadoop_libexec_dir}}\n\n# Mostly required for hadoop 2.0\nexport JAVA_LIBRARY_PATH=${JAVA_LIBRARY_PATH}\n\nexport HADOOP_OPTS=\"-Dhdp.version=$HDP_VERSION $HADOOP_
OPTS\"",
+ "hdfs_user": "hdfs",
+ "namenode_opt_newsize": "256m",
+ "dfs.datanode.data.dir.mount.file": "/etc/hadoop/conf/dfs_data_dir_mount.hist",
+ "hadoop_root_logger": "INFO,RFA",
+ "hadoop_heapsize": "1024",
+ "namenode_opt_maxpermsize": "256m",
+ "namenode_opt_permsize": "128m"
+ },
+ "spark-metrics-properties": {
+ "content": "\n# syntax: [instance].sink|source.[name].[options]=[value]\n\n# This file configures Spark's internal metrics system. The metrics system is\n# divided into instances which correspond to internal components.\n# Each instance can be configured to report its metrics to one or more sinks.\n# Accepted values for [instance] are \"master\", \"worker\", \"executor\", \"driver\",\n# and \"applications\". A wild card \"*\" can be used as an instance name, in\n# which case all instances will inherit the supplied property.\n#\n# Within an instance, a \"source\" specifies a particular set of grouped metrics.\n# there are two kinds of sources:\n# 1. Spark internal sources, like MasterSource, WorkerSource, etc, which will\n# collect a Spark component's internal state. Each instance is paired with a\n# Spark source that is added automatically.\n# 2. Common sources, like JvmSource, which will collect low level state.\n# These can be added through configuration options and ar
e then loaded\n# using reflection.\n#\n# A \"sink\" specifies where metrics are delivered to. Each instance can be\n# assigned one or more sinks.\n#\n# The sink|source field specifies whether the property relates to a sink or\n# source.\n#\n# The [name] field specifies the name of source or sink.\n#\n# The [options] field is the specific property of this source or sink. The\n# source or sink is responsible for parsing this property.\n#\n# Notes:\n# 1. To add a new sink, set the \"class\" option to a fully qualified class\n# name (see examples below).\n# 2. Some sinks involve a polling period. The minimum allowed polling period\n# is 1 second.\n# 3. Wild card properties can be overridden by more specific properties.\n# For example, master.sink.console.period takes precedence over\n# *.sink.console.period.\n# 4. A metrics specific configuration\n# \"spark.metrics.conf=${SPARK_HOME}/conf/metrics.properties\" should be\n# added to Java properties using -Dspark.metrics.conf=xxx if you wa
nt to\n# customize metrics system. You can also put the file in ${SPARK_HOME}/conf\n# and it will be loaded automatically.\n# 5. MetricsServlet is added by default as a sink in master, worker and client\n# driver, you can send http request \"/metrics/json\" to get a snapshot of all the\n# registered metrics in json format. For master, requests \"/metrics/master/json\" and\n# \"/metrics/applications/json\" can be sent seperately to get metrics snapshot of\n# instance master and applications. MetricsServlet may not be configured by self.\n#\n\n## List of available sinks and their properties.\n\n# org.apache.spark.metrics.sink.ConsoleSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n\n# org.apache.spark.metrics.sink.CSVSink\n# Name: Default: Description:\n# period 10 Poll period\n# unit seconds Units of poll period\n# directory /tmp Where to store CSV files\n\n# org.apache.spark.metrics.sink.GangliaSink\n# Name: Default: Description:\n#
host NONE Hostname or multicast group of Ganglia server\n# port NONE Port of Ganglia server(s)\n# period 10 Poll period\n# unit seconds Units of poll period\n# ttl 1 TTL of messages sent by Ganglia\n# mode multicast Ganglia network mode ('unicast' or 'multicast')\n\n# org.apache.spark.metrics.sink.JmxSink\n\n# org.apache.spark.metrics.sink.MetricsServlet\n# Name: Default: Description:\n# path VARIES* Path prefix from the web server root\n# sample false Whether to show entire set of samples for histograms ('false' or 'true')\n#\n# * Default path is /metrics/json for all instances except the master. The master has two paths:\n# /metrics/aplications/json # App information\n# /metrics/master/json # Master information\n\n# org.apache.spark.metrics.sink.GraphiteSink\n# Name: Default: Description:\n# host NONE Hostname of Graphite server\n# port NONE Port of Graphite server\n# period 10 Poll period\n# unit seconds Units of poll period\n# prefix EMPTY STRING Prefix to prepend to metric name
\n\n## Examples\n# Enable JmxSink for all instances by class name\n#*.sink.jmx.class=org.apache.spark.metrics.sink.JmxSink\n\n# Enable ConsoleSink for all instances by class name\n#*.sink.console.class=org.apache.spark.metrics.sink.ConsoleSink\n\n# Polling period for ConsoleSink\n#*.sink.console.period=10\n\n#*.sink.console.unit=seconds\n\n# Master instance overlap polling period\n#master.sink.console.period=15\n\n#master.sink.console.unit=seconds\n\n# Enable CsvSink for all instances\n#*.sink.csv.class=org.apache.spark.metrics.sink.CsvSink\n\n# Polling period for CsvSink\n#*.sink.csv.period=1\n\n#*.sink.csv.unit=minutes\n\n# Polling directory for CsvSink\n#*.sink.csv.directory=/tmp/\n\n# Worker instance overlap polling period\n#worker.sink.csv.period=10\n\n#worker.sink.csv.unit=minutes\n\n# Enable jvm source for instance master, worker, driver and executor\n#master.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#worker.source.jvm.class=org.apache.spark.metrics.source
.JvmSource\n\n#driver.source.jvm.class=org.apache.spark.metrics.source.JvmSource\n\n#executor.source.jvm.class=org.apache.spark.metrics.source.JvmSource"
+ },
"slider-client": {
"slider.yarn.queue": "default"
},