You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/11/16 05:37:31 UTC
ambari git commit: AMBARI-13899. Remove hdp.version configuration in
Ambari Spark definition (Saisai Shao via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk 66e90bcfa -> cacbb5532
AMBARI-13899. Remove hdp.version configuration in Ambari Spark definition (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/cacbb553
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/cacbb553
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/cacbb553
Branch: refs/heads/trunk
Commit: cacbb5532611131ebdc6f8ca4bc6a71d6998f7db
Parents: 66e90bc
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Nov 15 20:37:20 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Nov 15 20:37:20 2015 -0800
----------------------------------------------------------------------
.../1.2.0.2.2/configuration/spark-defaults.xml | 22 --------------
.../configuration/spark-javaopts-properties.xml | 30 --------------------
.../SPARK/1.2.0.2.2/metainfo.xml | 1 -
.../SPARK/1.2.0.2.2/package/scripts/params.py | 15 ----------
.../1.2.0.2.2/package/scripts/setup_spark.py | 14 +++------
.../SPARK/1.3.1.2.3/metainfo.xml | 1 -
.../SPARK/1.4.1.2.3/metainfo.xml | 1 -
.../configuration/spark-thrift-sparkconf.xml | 16 -----------
.../stacks/HDP/2.3/services/SPARK/metainfo.xml | 1 -
.../stacks/2.2/SPARK/test_job_history_server.py | 13 ++-------
.../stacks/2.2/SPARK/test_spark_client.py | 12 ++------
.../2.3/SPARK/test_spark_thrift_server.py | 5 ----
12 files changed, 8 insertions(+), 123 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
index 6def8b2..b507d5e 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
@@ -116,28 +116,6 @@
</property>
<property>
- <name>spark.driver.extraJavaOptions</name>
- <value>-Dhdp.version={{hdp_full_version}}</value>
- <description>
- Specifies parameters that are passed to the JVM of the Spark driver.
- </description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
-
- <property>
- <name>spark.yarn.am.extraJavaOptions</name>
- <value>-Dhdp.version={{hdp_full_version}}</value>
- <description>
- Specifies the parameters that are passed to the JVM of the Spark Application Master.
- </description>
- <value-attributes>
- <empty-value-valid>true</empty-value-valid>
- </value-attributes>
- </property>
-
- <property>
<name>spark.history.kerberos.principal</name>
<value>none</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
deleted file mode 100644
index f48d79b..0000000
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-javaopts-properties.xml
+++ /dev/null
@@ -1,30 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-<configuration supports_final="true">
- <property>
- <name>content</name>
- <description>Spark-javaopts-properties</description>
- <value> </value>
- <value-attributes>
- <show-property-name>false</show-property-name>
- </value-attributes>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
index f84fb73..61016ab 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/metainfo.xml
@@ -131,7 +131,6 @@
<config-type>spark-env</config-type>
<config-type>spark-log4j-properties</config-type>
<config-type>spark-metrics-properties</config-type>
- <config-type>spark-javaopts-properties</config-type>
</configuration-dependencies>
<commandScript>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index 7706650..985c2e3 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -112,25 +112,10 @@ spark_history_ui_port = config['configurations']['spark-defaults']['spark.histor
spark_env_sh = config['configurations']['spark-env']['content']
spark_log4j_properties = config['configurations']['spark-log4j-properties']['content']
spark_metrics_properties = config['configurations']['spark-metrics-properties']['content']
-spark_javaopts_properties = config['configurations']['spark-javaopts-properties']['content']
hive_server_host = default("/clusterHostInfo/hive_server_host", [])
is_hive_installed = not len(hive_server_host) == 0
-hdp_full_version = functions.get_hdp_version('spark-client')
-
-spark_driver_extraJavaOptions = str(config['configurations']['spark-defaults']['spark.driver.extraJavaOptions'])
-if spark_driver_extraJavaOptions.find('-Dhdp.version') == -1:
- spark_driver_extraJavaOptions = spark_driver_extraJavaOptions + ' -Dhdp.version=' + str(hdp_full_version)
-
-spark_yarn_am_extraJavaOptions = str(config['configurations']['spark-defaults']['spark.yarn.am.extraJavaOptions'])
-if spark_yarn_am_extraJavaOptions.find('-Dhdp.version') == -1:
- spark_yarn_am_extraJavaOptions = spark_yarn_am_extraJavaOptions + ' -Dhdp.version=' + str(hdp_full_version)
-
-spark_javaopts_properties = str(spark_javaopts_properties)
-if spark_javaopts_properties.find('-Dhdp.version') == -1:
- spark_javaopts_properties = spark_javaopts_properties+ ' -Dhdp.version=' + str(hdp_full_version)
-
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
spark_kerberos_keytab = config['configurations']['spark-defaults']['spark.history.kerberos.keytab']
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
index 9969a9b..debed40 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/setup_spark.py
@@ -44,12 +44,12 @@ def setup_spark(env, type, action = None):
mode=0775
)
params.HdfsResource(None, action="execute")
-
+
PropertiesFile(format("{spark_conf}/spark-defaults.conf"),
properties = params.config['configurations']['spark-defaults'],
- key_value_delimiter = " ",
+ key_value_delimiter = " ",
owner=params.spark_user,
- group=params.spark_group,
+ group=params.spark_group,
)
# create spark-env.sh in etc/conf dir
@@ -73,12 +73,6 @@ def setup_spark(env, type, action = None):
content=InlineTemplate(params.spark_metrics_properties)
)
- File(os.path.join(params.spark_conf, 'java-opts'),
- owner=params.spark_user,
- group=params.spark_group,
- content=params.spark_javaopts_properties
- )
-
if params.is_hive_installed:
XmlConfig("hive-site.xml",
conf_dir=params.spark_conf,
@@ -92,5 +86,5 @@ def setup_spark(env, type, action = None):
and 'spark-thrift-sparkconf' in params.config['configurations']:
PropertiesFile(params.spark_thrift_server_conf_file,
properties = params.config['configurations']['spark-thrift-sparkconf'],
- key_value_delimiter = " ",
+ key_value_delimiter = " ",
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
index 306d16c..2869f75 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.3.1.2.3/metainfo.xml
@@ -125,7 +125,6 @@
<config-type>spark-env</config-type>
<config-type>spark-log4j-properties</config-type>
<config-type>spark-metrics-properties</config-type>
- <config-type>spark-javaopts-properties</config-type>
</configuration-dependencies>
<commandScript>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
index 839bbac..0923dda 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.4.1.2.3/metainfo.xml
@@ -68,7 +68,6 @@
<config-type>spark-env</config-type>
<config-type>spark-log4j-properties</config-type>
<config-type>spark-metrics-properties</config-type>
- <config-type>spark-javaopts-properties</config-type>
<config-type>spark-thrift-sparkconf</config-type>
<config-type>spark-hive-site-override</config-type>
</configuration-dependencies>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
index 41d2e21..b5742ea 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -92,22 +92,6 @@
</property>
<property>
- <name>spark.driver.extraJavaOptions</name>
- <value>-Dhdp.version={{hdp_full_version}}</value>
- <description>
- Specifies parameters that are passed to the JVM of the Spark driver.
- </description>
- </property>
-
- <property>
- <name>spark.yarn.am.extraJavaOptions</name>
- <value>-Dhdp.version={{hdp_full_version}}</value>
- <description>
- Specifies the parameters that are passed to the JVM of the Spark Application Master.
- </description>
- </property>
-
- <property>
<name>spark.yarn.max.executor.failures</name>
<value>3</value>
<description>The maximum number of executor failures before failing the application.</description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
index d86afdf..2c9680c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
@@ -58,7 +58,6 @@
<config-type>spark-env</config-type>
<config-type>spark-log4j-properties</config-type>
<config-type>spark-metrics-properties</config-type>
- <config-type>spark-javaopts-properties</config-type>
<config-type>spark-thrift-sparkconf</config-type>
<config-type>spark-hive-site-override</config-type>
</configuration-dependencies>
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
index 2a45c09..90d3f0a 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_job_history_server.py
@@ -215,12 +215,7 @@ class TestJobHistoryServer(RMFTestCase):
owner = 'spark',
group = 'spark',
)
- self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
- content = ' -Dhdp.version=2.3.0.0-1597',
- owner = 'spark',
- group = 'spark',
- )
-
+
def assert_configure_secured(self):
self.assertResourceCalled('Directory', '/var/run/spark',
owner = 'spark',
@@ -282,11 +277,7 @@ class TestJobHistoryServer(RMFTestCase):
owner = 'spark',
group = 'spark',
)
- self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
- content = ' -Dhdp.version=2.3.0.0-1597',
- owner = 'spark',
- group = 'spark',
- )
+
@patch("resource_management.libraries.functions.copy_tarball.copy_to_hdfs")
def test_pre_upgrade_restart_23(self, copy_to_hdfs_mock):
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
index 9516d2f..798feb8 100644
--- a/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
+++ b/ambari-server/src/test/python/stacks/2.2/SPARK/test_spark_client.py
@@ -83,11 +83,7 @@ class TestSparkClient(RMFTestCase):
owner = 'spark',
group = 'spark',
)
- self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
- content = ' -Dhdp.version=2.3.0.0-1597',
- owner = 'spark',
- group = 'spark',
- )
+
def assert_configure_secured(self):
self.assertResourceCalled('Directory', '/var/run/spark',
@@ -121,11 +117,7 @@ class TestSparkClient(RMFTestCase):
owner = 'spark',
group = 'spark',
)
- self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
- content = ' -Dhdp.version=2.3.0.0-1597',
- owner = 'spark',
- group = 'spark',
- )
+
def test_pre_upgrade_restart_23(self):
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
http://git-wip-us.apache.org/repos/asf/ambari/blob/cacbb553/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
index 0fcfbf0..05745ae 100644
--- a/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/SPARK/test_spark_thrift_server.py
@@ -138,11 +138,6 @@ class TestSparkThriftServer(RMFTestCase):
owner = 'spark',
group = 'spark',
)
- self.assertResourceCalled('File', '/usr/hdp/current/spark-client/conf/java-opts',
- content = ' -Dhdp.version=2.3.2.0-1597',
- owner = 'spark',
- group = 'spark',
- )
self.assertResourceCalled('PropertiesFile', '/usr/hdp/current/spark-client/conf/spark-thrift-sparkconf.conf',
key_value_delimiter = ' ',
properties = self.getConfig()['configurations']['spark-thrift-sparkconf']