You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/01/19 16:03:31 UTC
[2/2] ambari git commit: AMBARI-14721. yarn-site properties for Spark
Shuffle Aux should be added unconditionally (aonishuk)
AMBARI-14721. yarn-site properties for Spark Shuffle Aux should be added unconditionally (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ce77df08
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ce77df08
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ce77df08
Branch: refs/heads/branch-2.2
Commit: ce77df08e9da72cffadb7ece2734cfbf31bd067b
Parents: bd4892b
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Tue Jan 19 17:03:18 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Tue Jan 19 17:03:18 2016 +0200
----------------------------------------------------------------------
.../upgrades/SparkShufflePropertyConfig.java | 14 ++--
.../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 2 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 2 +-
.../HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml | 2 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.4.xml | 2 +-
.../services/YARN/configuration/yarn-site.xml | 14 +---
.../stacks/HDP/2.4/services/stack_advisor.py | 78 +-------------------
.../SparkShufflePropertyConfigTest.java | 43 +----------
8 files changed, 12 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
index efeb4a0..299a373 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfig.java
@@ -68,9 +68,6 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
return createCommandReport(0, HostRoleStatus.FAILED,"{}",
String.format("Source type %s not found", YARN_SITE_CONFIG_TYPE), "");
}
-
- try {
- cluster.getService("SPARK"); // check if SPARK service present
Map<String, String> yarnSiteProperties = yarnSiteConfig.getProperties();
final List<String> auxSevices;
@@ -82,7 +79,11 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
} else {
auxSevices = new ArrayList<>();
}
- auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+
+ // check if spark is not already in the list
+ if (!auxSevices.contains(SPARK_SHUFFLE_AUX_STR)) {
+ auxSevices.add(SPARK_SHUFFLE_AUX_STR);
+ }
newAuxServices = StringUtils.join(auxSevices, ",");
yarnSiteProperties.put(YARN_NODEMANAGER_AUX_SERVICES, newAuxServices);
@@ -94,10 +95,5 @@ public class SparkShufflePropertyConfig extends AbstractServerAction {
String.format("%s was set from %s to %s. %s was set to %s",
YARN_NODEMANAGER_AUX_SERVICES, oldAuxServices, newAuxServices,
YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE, YARN_NODEMANAGER_AUX_SERVICES_SPARK_SHUFFLE_CLASS_VALUE), "");
-
- } catch (ServiceNotFoundException e) {
- return createCommandReport(0, HostRoleStatus.COMPLETED, "{}",
- String.format("%s not updated as no SPARK service present on the cluster", YARN_NODEMANAGER_AUX_SERVICES), "");
- }
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index ff7e032..c5d74e2 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -395,7 +395,7 @@
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
</execute-stage>
- <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark Shuffle">
<task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</execute-stage>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index 34e4848..6b5bcb1 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -566,7 +566,7 @@
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
- <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</pre-upgrade>
<upgrade>
<task xsi:type="restart-task"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
index f58eb79..a3880b3 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/nonrolling-upgrade-2.4.xml
@@ -269,7 +269,7 @@
<supports-auto-skip-failure>false</supports-auto-skip-failure>
<execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
- <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</execute-stage>
<execute-stage service="TEZ" component="TEZ_CLIENT" title="Apply config changes for Tez">
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
index 573713c..c15769b 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.4.xml
@@ -488,7 +488,7 @@
<component name="RESOURCEMANAGER">
<pre-upgrade>
- <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark Shuffle" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</pre-upgrade>
<pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
index 134d9b0..5e8eae6 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/YARN/configuration/yarn-site.xml
@@ -20,26 +20,14 @@
<configuration supports_final="true">
<property>
<name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
+ <value>mapreduce_shuffle,spark_shuffle</value>
<description>Auxilliary services of NodeManager. A valid service name should only contain a-zA-Z0-9_ and can
not start with numbers</description>
- <depends-on>
- <property>
- <type>spark-default</type>
- <name>spark.yarn.queue</name>
- </property>
- </depends-on>
</property>
<property>
<name>yarn.nodemanager.aux-services.spark_shuffle.class</name>
<value>org.apache.spark.network.yarn.YarnShuffleService</value>
<description>The auxiliary service class to use for Spark</description>
- <depends-on>
- <property>
- <type>spark-default</type>
- <name>spark.yarn.queue</name>
- </property>
- </depends-on>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
index 3031ab3..985c101 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/services/stack_advisor.py
@@ -19,80 +19,4 @@ limitations under the License.
class HDP24StackAdvisor(HDP23StackAdvisor):
- def getServiceConfigurationRecommenderDict(self):
- parent_recommend_conf_dict = super(HDP24StackAdvisor, self).getServiceConfigurationRecommenderDict()
- child_recommend_conf_dict = {
- "YARN": self.recommendYARNConfigurations
- }
- parent_recommend_conf_dict.update(child_recommend_conf_dict)
- return parent_recommend_conf_dict
-
- def getServiceConfigurationValidators(self):
- parentValidators = super(HDP24StackAdvisor, self).getServiceConfigurationValidators()
- childValidators = {
- "YARN": {"yarn-site": self.validateYARNConfigurations}
- }
- self.mergeValidators(parentValidators, childValidators)
- return parentValidators
-
- def recommendYARNConfigurations(self, configurations, clusterData, services, hosts):
- super(HDP24StackAdvisor, self).recommendYARNConfigurations(configurations, clusterData, services, hosts)
-
- yarn_site_config = "yarn-site"
- properties = services["configurations"] if yarn_site_config in services["configurations"] else []
- yarn_site_properties = properties[yarn_site_config]["properties"] if yarn_site_config in properties and \
- "properties" in properties[yarn_site_config] else []
- put_yarn_site_property = self.putProperty(configurations, yarn_site_config, services)
- put_yarn_site_property_attributes = self.putPropertyAttribute(configurations, yarn_site_config)
- services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
- if 'SPARK' in services_list:
- if "yarn.nodemanager.aux-services" in yarn_site_properties:
- aux_services = yarn_site_properties["yarn.nodemanager.aux-services"].split(",")
- if "spark_shuffle" not in aux_services:
- aux_services.append("spark_shuffle")
- put_yarn_site_property("yarn.nodemanager.aux-services", ",".join(aux_services))
- else:
- put_yarn_site_property("yarn.nodemanager.aux-services", "spark_shuffle")
-
- put_yarn_site_property("yarn.nodemanager.aux-services.spark_shuffle.class",
- "org.apache.spark.network.yarn.YarnShuffleService")
- else:
- put_yarn_site_property_attributes("yarn.nodemanager.aux-services.spark_shuffle.class", "delete", "true")
-
- def validateYARNConfigurations(self, properties, recommendedDefaults, configurations, services, hosts):
- yarn_site = properties
- validationItems = []
- services_list = [service["StackServices"]["service_name"] for service in services["services"]]
-
- if "SPARK" in services_list and 'YARN' in services_list:
- # yarn.nodemanager.aux-services = ...,spark_shuffle,....
- # yarn.nodemanager.aux-services.spark_shuffle.class = <not set>
- if "yarn.nodemanager.aux-services" in yarn_site \
- and "spark_shuffle" in yarn_site["yarn.nodemanager.aux-services"].lower() \
- and "yarn.nodemanager.aux-services.spark_shuffle.class" not in yarn_site:
- validationItems.append({
- "config-name": "yarn.nodemanager.aux-services.spark_shuffle.class",
- "item": self.getErrorItem("If spark_shuffle is listed in the aux-services, property value for " +
- "yarn.nodemanager.aux-services.spark_shuffle.class need to be set")
- })
-
- # yarn.nodemanager.aux-services = <not set>
- # yarn.nodemanager.aux-services.spark_shuffle.class = is set
- spark_aux_service_warning = False
- if "yarn.nodemanager.aux-services" in yarn_site and "spark_shuffle" not in yarn_site[
- "yarn.nodemanager.aux-services"].lower() \
- and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
- spark_aux_service_warning = True
-
- if "yarn.nodemanager.aux-services" not in yarn_site and "yarn.nodemanager.aux-services.spark_shuffle.class" in yarn_site:
- spark_aux_service_warning = True
-
- if spark_aux_service_warning:
- validationItems.append({
- "config-name": "yarn.nodemanager.aux-services",
- "item": self.getWarnItem("If yarn.nodemanager.aux-services.spark_shuffle.class is set, probably " +
- "aux-services property need to be updated to enable spark_shuffle")
- })
-
- return self.toConfigurationValidationProblems(validationItems, "yarn-site")
+ pass
http://git-wip-us.apache.org/repos/asf/ambari/blob/ce77df08/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
index edba6f0..e65a824 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
@@ -22,7 +22,6 @@ import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.expect;
import static org.easymock.EasyMock.replay;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -30,7 +29,6 @@ import java.lang.reflect.Field;
import java.util.HashMap;
import java.util.Map;
-import org.apache.ambari.server.ServiceNotFoundException;
import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
import org.apache.ambari.server.actionmanager.HostRoleCommand;
import org.apache.ambari.server.agent.CommandReport;
@@ -39,7 +37,6 @@ import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.ConfigImpl;
-import org.apache.ambari.server.state.Service;
import org.easymock.EasyMock;
import org.junit.Before;
import org.junit.Test;
@@ -94,20 +91,16 @@ public class SparkShufflePropertyConfigTest {
}
@Test
- public void testSparkPresent() throws Exception {
+ public void testAction() throws Exception {
ExecutionCommand executionCommand = new ExecutionCommand();
Map<String, String> commandParams = new HashMap<>();
HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
- Service sparkService = EasyMock.createMock(Service.class);
SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
commandParams.put("clusterName", "c1");
executionCommand.setCommandParams(commandParams);
executionCommand.setClusterName("c1");
- // test calculations when spark service present
- expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce();
-
expect(hrc.getRequestId()).andReturn(1L).anyTimes();
expect(hrc.getStageId()).andReturn(2L).anyTimes();
expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
@@ -131,39 +124,5 @@ public class SparkShufflePropertyConfigTest {
assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class"));
}
- @Test
- public void testSparkNotPresent() throws Exception{
- SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
- Map<String, String> commandParams = new HashMap<>();
- ExecutionCommand executionCommand = new ExecutionCommand();
- HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
- ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK");
- commandParams.put("clusterName", "c1");
- executionCommand.setCommandParams(commandParams);
- executionCommand.setClusterName("c1");
-
- //no spark service present
- expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce();
-
- expect(hrc.getRequestId()).andReturn(1L).anyTimes();
- expect(hrc.getStageId()).andReturn(2L).anyTimes();
- expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
- replay(cluster, hrc);
-
- clusterField.set(action, m_clusters);
- action.setExecutionCommand(executionCommand);
- action.setHostRoleCommand(hrc);
-
- CommandReport report = action.execute(null);
- assertNotNull(report);
-
- Cluster c = m_clusters.getCluster("c1");
- Config config = c.getDesiredConfigByType("yarn-site");
- Map<String, String> map = config.getProperties();
-
- assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
- assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
- assertEquals("some_service", map.get("yarn.nodemanager.aux-services"));
- }
}