You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sm...@apache.org on 2015/11/09 02:29:22 UTC
[1/3] ambari git commit: AMBARI-13650. Bump the Ambari Spark
definition to Spark 1.5.1 (Saisai Shao via smohanty)
Repository: ambari
Updated Branches:
refs/heads/trunk 66e2eb424 -> 618d843c1
AMBARI-13650. Bump the Ambari Spark definition to Spark 1.5.1 (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db224500
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db224500
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db224500
Branch: refs/heads/trunk
Commit: db224500f253f2bd6c842f9097efe572943001cf
Parents: 66e2eb4
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Nov 8 17:13:35 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Nov 8 17:13:35 2015 -0800
----------------------------------------------------------------------
.../1.2.0.2.2/configuration/spark-defaults.xml | 17 ++-----
.../SPARK/1.5.1.2.3/metainfo.xml | 50 ++++++++++++++++++++
.../configuration/spark-thrift-sparkconf.xml | 13 +----
.../stacks/HDP/2.3/services/SPARK/metainfo.xml | 45 ++----------------
4 files changed, 60 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
index 8b13779..05d1a3a 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
@@ -42,15 +42,6 @@
</property>
<property>
- <name>spark.yarn.applicationMaster.waitTries</name>
- <value>10</value>
- <description>
- Set the number of times the ApplicationMaster waits for the the Spark master and then
- also the number of tries it waits for the SparkContext to be initialized.
- </description>
- </property>
-
- <property>
<name>spark.yarn.scheduler.heartbeat.interval-ms</name>
<value>5000</value>
<description>
@@ -95,7 +86,7 @@
<name>spark.yarn.preserve.staging.files</name>
<value>false</value>
<description>
- Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the
+ Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the
end of the job rather then delete them.
</description>
</property>
@@ -161,19 +152,19 @@
Location of the kerberos keytab file for the Spark History Server.
</description>
</property>
-
+
<property>
<name>spark.history.provider</name>
<value>org.apache.spark.deploy.yarn.history.YarnHistoryProvider</value>
<description>Name of history provider class</description>
</property>
-
+
<property>
<name>spark.yarn.historyServer.address</name>
<value>{{spark_history_server_host}}:{{spark_history_ui_port}}</value>
<description>The address of the Spark history server (i.e. host.com:18080). The address should not contain a scheme (http://). Defaults to not being set since the history server is an optional service. This address is given to the YARN ResourceManager when the Spark application finishes to link the application from the ResourceManager UI to the Spark history server UI.</description>
</property>
-
+
<property>
<name>spark.yarn.max.executor.failures</name>
<value>3</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
new file mode 100644
index 0000000..190d892
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.5.1.2.3/metainfo.xml
@@ -0,0 +1,50 @@
+<?xml version="1.0"?>
+<!--Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SPARK</name>
+ <extends>common-services/SPARK/1.4.1.2.3</extends>
+ <displayName>Spark</displayName>
+ <comment>Apache Spark is a fast and general engine for large-scale data processing.</comment>
+ <version>1.5.1.2.3</version>
+ <!-- No new components added for 1.5.1 -->
+ <configuration-dependencies>
+ <config-type>spark-defaults</config-type>
+ <config-type>spark-env</config-type>
+ <config-type>spark-log4j-properties</config-type>
+ <config-type>spark-metrics-properties</config-type>
+ <config-type>spark-javaopts-properties</config-type>
+ <config-type>spark-thrift-sparkconf</config-type>
+ <config-type>spark-hive-site-override</config-type>
+ </configuration-dependencies>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
index c42841f..41d2e21 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/configuration/spark-thrift-sparkconf.xml
@@ -42,15 +42,6 @@
</property>
<property>
- <name>spark.yarn.applicationMaster.waitTries</name>
- <value>10</value>
- <description>
- Set the number of times the ApplicationMaster waits for the the Spark master and then
- also the number of tries it waits for the SparkContext to be initialized.
- </description>
- </property>
-
- <property>
<name>spark.yarn.scheduler.heartbeat.interval-ms</name>
<value>5000</value>
<description>
@@ -95,7 +86,7 @@
<name>spark.yarn.preserve.staging.files</name>
<value>false</value>
<description>
- Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the
+ Set to true to preserve the staged files (Spark jar, app jar, distributed cache files) at the
end of the job rather then delete them.
</description>
</property>
@@ -115,7 +106,7 @@
Specifies the parameters that are passed to the JVM of the Spark Application Master.
</description>
</property>
-
+
<property>
<name>spark.yarn.max.executor.failures</name>
<value>3</value>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db224500/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
index 14161b4..0e20794 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/SPARK/metainfo.xml
@@ -23,8 +23,8 @@
<services>
<service>
<name>SPARK</name>
- <version>1.4.1.2.3</version>
- <extends>common-services/SPARK/1.3.1.2.3</extends>
+ <version>1.5.1.2.3</version>
+ <extends>common-services/SPARK/1.4.1.2.3</extends>
<osSpecifics>
<osSpecific>
<osFamily>redhat7,redhat6,suse11</osFamily>
@@ -52,44 +52,7 @@
<requiredServices>
<service>YARN</service>
</requiredServices>
- <components>
- <component>
- <name>SPARK_THRIFTSERVER</name>
- <displayName>Spark Thrift Server</displayName>
- <deleted>true</deleted>
- <category>MASTER</category>
- <cardinality>0+</cardinality>
- <versionAdvertised>true</versionAdvertised>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>YARN/YARN_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/spark_thrift_server.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
+ <!-- No new components compared to 1.4.1 -->
<configuration-dependencies>
<config-type>spark-defaults</config-type>
<config-type>spark-env</config-type>
@@ -98,7 +61,7 @@
<config-type>spark-javaopts-properties</config-type>
<config-type>spark-thrift-sparkconf</config-type>
<config-type>spark-hive-site-override</config-type>
- </configuration-dependencies>
+ </configuration-dependencies>
</service>
</services>
</metainfo>
[2/3] ambari git commit: AMBARI-13728. Fix Spark History server not
work problem (Saisai Shao via smohanty)
Posted by sm...@apache.org.
AMBARI-13728. Fix Spark History server not work problem (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a5d5dc99
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a5d5dc99
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a5d5dc99
Branch: refs/heads/trunk
Commit: a5d5dc99a80cf4f8d4639c829f84e7112d95d0dd
Parents: db22450
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Nov 8 17:20:34 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Nov 8 17:20:34 2015 -0800
----------------------------------------------------------------------
.../SPARK/1.2.0.2.2/configuration/spark-defaults.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/a5d5dc99/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
index 05d1a3a..6def8b2 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/configuration/spark-defaults.xml
@@ -101,7 +101,7 @@
<property>
<name>spark.history.provider</name>
- <value>org.apache.spark.deploy.yarn.history.YarnHistoryProvider</value>
+ <value>org.apache.spark.deploy.history.yarn.server.YarnHistoryProvider</value>
<description>
Name of the class implementing the application history backend which publishes to YARN Application Timeline Service.
</description>
[3/3] ambari git commit: AMBARI-13729. Change the Spark thrift server
security configurations (Saisai Shao via smohanty)
Posted by sm...@apache.org.
AMBARI-13729. Change the Spark thrift server security configurations (Saisai Shao via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/618d843c
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/618d843c
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/618d843c
Branch: refs/heads/trunk
Commit: 618d843c1b8b436748117b9e3852f85ae802873b
Parents: a5d5dc9
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Nov 8 17:21:23 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Nov 8 17:21:23 2015 -0800
----------------------------------------------------------------------
.../common-services/SPARK/1.2.0.2.2/package/scripts/params.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/618d843c/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
index e5b2fb8..7706650 100644
--- a/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/SPARK/1.2.0.2.2/package/scripts/params.py
@@ -105,7 +105,6 @@ else:
spark_history_server_host = "localhost"
# spark-defaults params
-spark_hive_sec_authorization_enabled = "false"
spark_yarn_historyServer_address = default(spark_history_server_host, "localhost")
spark_history_ui_port = config['configurations']['spark-defaults']['spark.history.ui.port']
@@ -145,7 +144,7 @@ spark_hive_properties = {
# security settings
if security_enabled:
spark_principal = spark_kerberos_principal.replace('_HOST',spark_history_server_host.lower())
-
+
if is_hive_installed:
spark_hive_properties.update({
'hive.metastore.sasl.enabled': str(config['configurations']['hive-site']['hive.metastore.sasl.enabled']).lower(),
@@ -155,8 +154,7 @@ if security_enabled:
'hive.metastore.kerberos.principal': config['configurations']['hive-site']['hive.metastore.kerberos.principal'],
'hive.server2.authentication.kerberos.principal': config['configurations']['hive-site']['hive.server2.authentication.kerberos.principal'],
'hive.server2.authentication.kerberos.keytab': config['configurations']['hive-site']['hive.server2.authentication.kerberos.keytab'],
- 'hive.security.authorization.enabled': spark_hive_sec_authorization_enabled,
- 'hive.server2.enable.doAs': str(config['configurations']['hive-site']['hive.server2.enable.doAs']).lower()
+ 'hive.server2.authentication': config['configurations']['hive-site']['hive.server2.authentication'],
})
# thrift server support - available on HDP 2.3 or higher