You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mp...@apache.org on 2016/01/15 15:26:11 UTC
[2/2] ambari git commit: AMBARI-14669. Add yarn-site properties for
Spark Shuffle Aux services (2.2->2.4 upgrade path). (Dmytro Grinenko via
mpapirkovskyy)
AMBARI-14669. Add yarn-site properties for Spark Shuffle Aux services (2.2->2.4 upgrade path). (Dmytro Grinenko via mpapirkovskyy)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e0957d39
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e0957d39
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e0957d39
Branch: refs/heads/trunk
Commit: e0957d397a2780683cc3387c30359c68669a9343
Parents: 672fb60
Author: Myroslav Papirkovskyy <mp...@hortonworks.com>
Authored: Fri Jan 15 16:24:56 2016 +0200
Committer: Myroslav Papirkovskyy <mp...@hortonworks.com>
Committed: Fri Jan 15 16:25:41 2016 +0200
----------------------------------------------------------------------
.../HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml | 4 +
.../stacks/HDP/2.2/upgrades/upgrade-2.4.xml | 1 +
.../SparkShufflePropertyConfigTest.java | 169 +++++++++++++++++++
3 files changed, 174 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
index 1efeb6c..00b9e96 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/nonrolling-upgrade-2.4.xml
@@ -395,6 +395,10 @@
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
</execute-stage>
+ <execute-stage service="YARN" component="RESOURCEMANAGER" title="Calculating Yarn Properties for Spark">
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
+ </execute-stage>
+
<!--HBASE-->
<execute-stage service="HBASE" component="HBASE_MASTER" title="Apply config changes for HBase Master">
<task xsi:type="configure" id="hdp_2_3_0_0_hbase_master_adjust_phoenix_scheduler_factory"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
index acebdd8..7637830 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.4.xml
@@ -566,6 +566,7 @@
<task xsi:type="configure" id="hdp_2_3_0_0_yarn_rm_check_cs_root_max_capacity"/>
+ <task xsi:type="server_action" summary="Calculating Yarn Properties for Spark" class="org.apache.ambari.server.serveraction.upgrades.SparkShufflePropertyConfig" />
</pre-upgrade>
<upgrade>
<task xsi:type="restart-task"/>
http://git-wip-us.apache.org/repos/asf/ambari/blob/e0957d39/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
new file mode 100644
index 0000000..edba6f0
--- /dev/null
+++ b/ambari-server/src/test/java/org/apache/ambari/server/serveraction/upgrades/SparkShufflePropertyConfigTest.java
@@ -0,0 +1,169 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ambari.server.serveraction.upgrades;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.lang.reflect.Field;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.ambari.server.ServiceNotFoundException;
+import org.apache.ambari.server.actionmanager.ExecutionCommandWrapper;
+import org.apache.ambari.server.actionmanager.HostRoleCommand;
+import org.apache.ambari.server.agent.CommandReport;
+import org.apache.ambari.server.agent.ExecutionCommand;
+import org.apache.ambari.server.state.Cluster;
+import org.apache.ambari.server.state.Clusters;
+import org.apache.ambari.server.state.Config;
+import org.apache.ambari.server.state.ConfigImpl;
+import org.apache.ambari.server.state.Service;
+import org.easymock.EasyMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.inject.Injector;
+
+/**
+ * Tests upgrade-related server side actions for SparkShufflePropertyConfig
+ */
+public class SparkShufflePropertyConfigTest {
+ private Injector m_injector;
+ private Clusters m_clusters;
+ private Cluster cluster;
+ private Field clusterField;
+
+ @Before
+ public void setup() throws Exception {
+ m_injector = EasyMock.createMock(Injector.class);
+ m_clusters = EasyMock.createMock(Clusters.class);
+ cluster = EasyMock.createMock(Cluster.class);
+
+
+ Config adminConfig = new ConfigImpl("yarn-site") {
+ Map<String, String> mockProperties = new HashMap<String, String>() {{
+ put("yarn.nodemanager.aux-services", "some_service");
+ }};
+ @Override
+ public Map<String, String> getProperties() {
+ return mockProperties;
+ }
+
+ @Override
+ public void setProperties(Map<String, String> properties) {
+ mockProperties.putAll(properties);
+ }
+
+ @Override
+ public void persist(boolean newConfig) {
+ // no-op
+ }
+ };
+
+ expect(cluster.getDesiredConfigByType("yarn-site")).andReturn(adminConfig).atLeastOnce();
+
+ expect(m_clusters.getCluster((String) anyObject())).andReturn(cluster).anyTimes();
+ expect(m_injector.getInstance(Clusters.class)).andReturn(m_clusters).atLeastOnce();
+
+ replay(m_injector, m_clusters);
+
+ clusterField = SparkShufflePropertyConfig.class.getDeclaredField("clusters");
+ clusterField.setAccessible(true);
+ }
+
+ @Test
+ public void testSparkPresent() throws Exception {
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ Map<String, String> commandParams = new HashMap<>();
+ HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+ Service sparkService = EasyMock.createMock(Service.class);
+ SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
+
+ commandParams.put("clusterName", "c1");
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName("c1");
+
+ // test calculations when spark service present
+ expect(cluster.getService("SPARK")).andReturn(sparkService).atLeastOnce();
+
+ expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+ expect(hrc.getStageId()).andReturn(2L).anyTimes();
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+ replay(cluster, hrc);
+
+ clusterField.set(action, m_clusters);
+ action.setExecutionCommand(executionCommand);
+ action.setHostRoleCommand(hrc);
+
+ CommandReport report = action.execute(null);
+ assertNotNull(report);
+
+ Cluster c = m_clusters.getCluster("c1");
+ Config config = c.getDesiredConfigByType("yarn-site");
+ Map<String, String> map = config.getProperties();
+
+ assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
+ assertTrue(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
+
+ assertEquals("some_service,spark_shuffle", map.get("yarn.nodemanager.aux-services"));
+ assertEquals("org.apache.spark.network.yarn.YarnShuffleService", map.get("yarn.nodemanager.aux-services.spark_shuffle.class"));
+ }
+
+ @Test
+ public void testSparkNotPresent() throws Exception{
+ SparkShufflePropertyConfig action = new SparkShufflePropertyConfig();
+ Map<String, String> commandParams = new HashMap<>();
+ ExecutionCommand executionCommand = new ExecutionCommand();
+ HostRoleCommand hrc = EasyMock.createMock(HostRoleCommand.class);
+ ServiceNotFoundException notFoundException = new ServiceNotFoundException("c1", "SPARK");
+
+ commandParams.put("clusterName", "c1");
+ executionCommand.setCommandParams(commandParams);
+ executionCommand.setClusterName("c1");
+
+ //no spark service present
+ expect(cluster.getService("SPARK")).andThrow(notFoundException).atLeastOnce();
+
+ expect(hrc.getRequestId()).andReturn(1L).anyTimes();
+ expect(hrc.getStageId()).andReturn(2L).anyTimes();
+ expect(hrc.getExecutionCommandWrapper()).andReturn(new ExecutionCommandWrapper(executionCommand)).anyTimes();
+ replay(cluster, hrc);
+
+ clusterField.set(action, m_clusters);
+ action.setExecutionCommand(executionCommand);
+ action.setHostRoleCommand(hrc);
+
+ CommandReport report = action.execute(null);
+ assertNotNull(report);
+
+ Cluster c = m_clusters.getCluster("c1");
+ Config config = c.getDesiredConfigByType("yarn-site");
+ Map<String, String> map = config.getProperties();
+
+ assertTrue(map.containsKey("yarn.nodemanager.aux-services"));
+ assertFalse(map.containsKey("yarn.nodemanager.aux-services.spark_shuffle.class"));
+ assertEquals("some_service", map.get("yarn.nodemanager.aux-services"));
+ }
+}