You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2013/04/04 01:16:42 UTC
svn commit: r1464231 - in /incubator/ambari/trunk: ./
ambari-server/src/main/java/org/apache/ambari/server/controller/
ambari-server/src/test/java/org/apache/ambari/server/controller/
Author: swagle
Date: Wed Apr 3 23:16:42 2013
New Revision: 1464231
URL: http://svn.apache.org/r1464231
Log:
AMBARI-1774. Ambari does not push the config updates to the client/gateway node. (swagle)
Modified:
incubator/ambari/trunk/CHANGES.txt
incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
Modified: incubator/ambari/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1464231&r1=1464230&r2=1464231&view=diff
==============================================================================
--- incubator/ambari/trunk/CHANGES.txt (original)
+++ incubator/ambari/trunk/CHANGES.txt Wed Apr 3 23:16:42 2013
@@ -544,6 +544,9 @@ Trunk (unreleased changes):
BUG FIXES
+ AMBARI-1774. Ambari does not push the config updates to the client/gateway
+ node. (swagle)
+
AMBARI-1780. POSTing new cluster returns 500 exception. (smohanty)
AMBARI-1781. Ambari Server should work with MySQL and Oracle where the
Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java?rev=1464231&r1=1464230&r2=1464231&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java (original)
+++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariManagementControllerImpl.java Wed Apr 3 23:16:42 2013
@@ -1748,34 +1748,13 @@ public class AmbariManagementControllerI
return null;
}
- private List<Stage> doStageCreation(Cluster cluster,
- Map<State, List<Service>> changedServices,
- Map<State, List<ServiceComponent>> changedComps,
- Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts,
- Map<String, String> requestParameters, String requestContext, boolean runSmokeTest)
- throws AmbariException {
+ private Set<String> getServicesForSmokeTests(Cluster cluster,
+ Map<State, List<Service>> changedServices,
+ Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts,
+ boolean runSmokeTest) throws AmbariException {
- // TODO handle different transitions?
- // Say HDFS to stopped and MR to started, what order should actions be done
- // in?
-
- // TODO additional validation?
- // verify all configs
- // verify all required components
-
- if ((changedServices == null || changedServices.isEmpty())
- && (changedComps == null || changedComps.isEmpty())
- && (changedScHosts == null || changedScHosts.isEmpty())) {
- return null;
- }
-
- Long requestId = null;
- List<Stage> stages = null;
+ Set<String> smokeTestServices = new HashSet<String>();
- Set<String> smokeTestServices =
- new HashSet<String>();
-
- // smoke test any service that goes from installed to started
if (changedServices != null) {
for (Entry<State, List<Service>> entry : changedServices.entrySet()) {
if (State.STARTED != entry.getKey()) {
@@ -1790,30 +1769,30 @@ public class AmbariManagementControllerI
}
Map<String, Map<String, Integer>> changedComponentCount =
- new HashMap<String, Map<String, Integer>>();
+ new HashMap<String, Map<String, Integer>>();
for (Map<State, List<ServiceComponentHost>> stateScHostMap :
changedScHosts.values()) {
for (Entry<State, List<ServiceComponentHost>> entry :
- stateScHostMap.entrySet()) {
+ stateScHostMap.entrySet()) {
if (State.STARTED != entry.getKey()) {
continue;
}
for (ServiceComponentHost sch : entry.getValue()) {
if (State.START_FAILED != sch.getState()
- && State.INSTALLED != sch.getState()) {
+ && State.INSTALLED != sch.getState()) {
continue;
}
if (!changedComponentCount.containsKey(sch.getServiceName())) {
changedComponentCount.put(sch.getServiceName(),
- new HashMap<String, Integer>());
+ new HashMap<String, Integer>());
}
if (!changedComponentCount.get(sch.getServiceName())
- .containsKey(sch.getServiceComponentName())) {
+ .containsKey(sch.getServiceComponentName())) {
changedComponentCount.get(sch.getServiceName())
- .put(sch.getServiceComponentName(), 1);
+ .put(sch.getServiceComponentName(), 1);
} else {
Integer i = changedComponentCount.get(sch.getServiceName())
- .get(sch.getServiceComponentName());
+ .get(sch.getServiceComponentName());
changedComponentCount.get(sch.getServiceName())
.put(sch.getServiceComponentName(), ++i);
}
@@ -1822,7 +1801,7 @@ public class AmbariManagementControllerI
}
for (Entry<String, Map<String, Integer>> entry :
- changedComponentCount.entrySet()) {
+ changedComponentCount.entrySet()) {
String serviceName = entry.getKey();
// smoke test service if more than one component is started
if (runSmokeTest && (entry.getValue().size() > 1)) {
@@ -1832,11 +1811,11 @@ public class AmbariManagementControllerI
for (String componentName :
changedComponentCount.get(serviceName).keySet()) {
ServiceComponent sc = cluster.getService(serviceName)
- .getServiceComponent(componentName);
+ .getServiceComponent(componentName);
StackId stackId = sc.getDesiredStackVersion();
ComponentInfo compInfo = ambariMetaInfo.getComponentCategory(
- stackId.getStackName(), stackId.getStackVersion(), serviceName,
- componentName);
+ stackId.getStackName(), stackId.getStackVersion(), serviceName,
+ componentName);
if (runSmokeTest && compInfo.isMaster()) {
smokeTestServices.add(serviceName);
}
@@ -1844,6 +1823,117 @@ public class AmbariManagementControllerI
// FIXME if master check if we need to run a smoke test for the master
}
}
+ return smokeTestServices;
+ }
+
+ private void addClientSchForReinstall(Cluster cluster,
+ Map<State, List<Service>> changedServices,
+ Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts)
+ throws AmbariException {
+
+ Set<String> services = new HashSet<String>();
+
+ if (changedServices != null) {
+ for (Entry<State, List<Service>> entry : changedServices.entrySet()) {
+ if (State.STARTED != entry.getKey()) {
+ continue;
+ }
+ for (Service s : entry.getValue()) {
+ if (State.INSTALLED == s.getDesiredState()) {
+ services.add(s.getName());
+ }
+ }
+ }
+ }
+
+ if (services == null || services.isEmpty())
+ return;
+
+ // Flatten changed Schs that are going to be Started
+ List<ServiceComponentHost> existingSchs = new
+ ArrayList<ServiceComponentHost>();
+ if (changedScHosts != null && !changedScHosts.isEmpty()) {
+ for (String sc : changedScHosts.keySet()) {
+ for (State s : changedScHosts.get(sc).keySet())
+ if (s == State.STARTED)
+ existingSchs.addAll(changedScHosts.get(sc).get(s));
+ }
+ }
+
+ Map<String, List<ServiceComponentHost>> clientSchs = new
+ HashMap<String, List<ServiceComponentHost>>();
+
+ for (String serviceName : services) {
+ Service s = cluster.getService(serviceName);
+ for (String component : s.getServiceComponents().keySet()) {
+ List<ServiceComponentHost> potentialHosts = null;
+ ServiceComponent sc = s.getServiceComponents().get(component);
+ if (sc.isClientComponent()) {
+ potentialHosts = new ArrayList<ServiceComponentHost>();
+ // Check if the Client components are in the list of changed hosts
+ if (existingSchs != null && !existingSchs.isEmpty()) {
+ for (ServiceComponentHost potentialSch : sc
+ .getServiceComponentHosts().values()) {
+ boolean addSch = true;
+ // Ignore the Sch if same service has changed on the same host
+ for (ServiceComponentHost existingSch : existingSchs) {
+ if (potentialSch.getHostName().equals(existingSch
+ .getHostName()) && potentialSch.getServiceName().equals
+ (existingSch.getServiceName())) {
+ addSch = false;
+ }
+ }
+ if (addSch)
+ potentialHosts.add(potentialSch);
+ }
+ }
+ }
+ if (potentialHosts != null && !potentialHosts.isEmpty()) {
+ clientSchs.put(sc.getName(), potentialHosts);
+ }
+ }
+ }
+ LOG.info("Client hosts for reinstall : " + clientSchs.size
+ ());
+
+ for (String sc : clientSchs.keySet()) {
+ Map<State, List<ServiceComponentHost>> schMap = new
+ HashMap<State, List<ServiceComponentHost>>();
+ schMap.put(State.INSTALLED, clientSchs.get(sc));
+ changedScHosts.put(sc, schMap);
+ }
+ }
+
+ private List<Stage> doStageCreation(Cluster cluster,
+ Map<State, List<Service>> changedServices,
+ Map<State, List<ServiceComponent>> changedComps,
+ Map<String, Map<State, List<ServiceComponentHost>>> changedScHosts,
+ Map<String, String> requestParameters, String requestContext, boolean runSmokeTest)
+ throws AmbariException {
+
+ // TODO handle different transitions?
+ // Say HDFS to stopped and MR to started, what order should actions be done
+ // in?
+
+ // TODO additional validation?
+ // verify all configs
+ // verify all required components
+
+ if ((changedServices == null || changedServices.isEmpty())
+ && (changedComps == null || changedComps.isEmpty())
+ && (changedScHosts == null || changedScHosts.isEmpty())) {
+ return null;
+ }
+
+ Long requestId = null;
+
+ // smoke test any service that goes from installed to started
+ Set<String> smokeTestServices = getServicesForSmokeTests(cluster,
+ changedServices, changedScHosts, runSmokeTest);
+
+ // Re-install client only hosts to reattach changed configs on service
+ // restart
+ addClientSchForReinstall(cluster, changedServices, changedScHosts);
if (!changedScHosts.isEmpty()
|| !smokeTestServices.isEmpty()) {
@@ -2530,8 +2620,8 @@ public class AmbariManagementControllerI
Cluster cluster = clusters.getCluster(clusterNames.iterator().next());
List<Stage> stages = doStageCreation(cluster, changedServices, changedComps,
- changedScHosts, null, requestProperties.get(REQUEST_CONTEXT_PROPERTY),
- runSmokeTest);
+ changedScHosts, null, requestProperties.get(REQUEST_CONTEXT_PROPERTY),
+ runSmokeTest);
persistStages(stages);
updateServiceStates(changedServices, changedComps, changedScHosts);
if (stages == null || stages.isEmpty()) {
Modified: incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java?rev=1464231&r1=1464230&r2=1464231&view=diff
==============================================================================
--- incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java (original)
+++ incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java Wed Apr 3 23:16:42 2013
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.fail;
-
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -31,11 +30,8 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-
import javax.persistence.EntityManager;
-
import junit.framework.Assert;
-
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.ClusterNotFoundException;
import org.apache.ambari.server.DuplicateResourceException;
@@ -79,7 +75,6 @@ import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.persist.PersistService;
@@ -192,6 +187,86 @@ public class AmbariManagementControllerT
controller.createHostComponents(requests);
}
+ private long stopService(String clusterName, String serviceName) throws
+ AmbariException {
+ ServiceRequest r = new ServiceRequest(clusterName, serviceName, null,
+ State.INSTALLED.toString());
+ Set<ServiceRequest> requests = new HashSet<ServiceRequest>();
+ requests.add(r);
+ Map<String, String> mapRequestProps = new HashMap<String, String>();
+ mapRequestProps.put("context", "Called from a test");
+ RequestStatusResponse resp = controller.updateServices(requests,
+ mapRequestProps, true);
+
+ Assert.assertEquals(State.INSTALLED,
+ clusters.getCluster(clusterName).getService(serviceName)
+ .getDesiredState());
+
+ // manually change live state to stopped as no running action manager
+ for (ServiceComponent sc :
+ clusters.getCluster(clusterName).getService(serviceName)
+ .getServiceComponents().values()) {
+ for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) {
+ sch.setState(State.INSTALLED);
+ }
+ }
+
+ return resp.getRequestId();
+ }
+
+ private long startService(String clusterName, String serviceName) throws
+ AmbariException {
+ ServiceRequest r = new ServiceRequest(clusterName, serviceName, null,
+ State.STARTED.toString());
+ Set<ServiceRequest> requests = new HashSet<ServiceRequest>();
+ requests.add(r);
+ Map<String, String> mapRequestProps = new HashMap<String, String>();
+ mapRequestProps.put("context", "Called from a test");
+ RequestStatusResponse resp = controller.updateServices(requests,
+ mapRequestProps, true);
+
+ Assert.assertEquals(State.STARTED,
+ clusters.getCluster(clusterName).getService(serviceName)
+ .getDesiredState());
+
+ // manually change live state to stopped as no running action manager
+ for (ServiceComponent sc :
+ clusters.getCluster(clusterName).getService(serviceName)
+ .getServiceComponents().values()) {
+ for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) {
+ sch.setState(State.STARTED);
+ }
+ }
+
+ return resp.getRequestId();
+ }
+
+ private long installService(String clusterName, String serviceName) throws
+ AmbariException {
+ ServiceRequest r = new ServiceRequest(clusterName, serviceName, null,
+ State.INSTALLED.toString());
+ Set<ServiceRequest> requests = new HashSet<ServiceRequest>();
+ requests.add(r);
+ Map<String, String> mapRequestProps = new HashMap<String, String>();
+ mapRequestProps.put("context", "Called from a test");
+ RequestStatusResponse resp = controller.updateServices(requests,
+ mapRequestProps, true);
+
+ Assert.assertEquals(State.INSTALLED,
+ clusters.getCluster(clusterName).getService(serviceName)
+ .getDesiredState());
+ // manually change live state to stopped as no running action manager
+ for (ServiceComponent sc :
+ clusters.getCluster(clusterName).getService(serviceName)
+ .getServiceComponents().values()) {
+ for (ServiceComponentHost sch : sc.getServiceComponentHosts().values()) {
+ sch.setState(State.INSTALLED);
+ }
+ }
+
+ return resp.getRequestId();
+ }
+
@Test
public void testCreateClusterSimple() throws AmbariException {
String clusterName = "foo1";
@@ -4294,6 +4369,148 @@ public class AmbariManagementControllerT
}
@Test
+ public void testReConfigureServiceClient() throws AmbariException {
+ String clusterName = "foo1";
+ createCluster(clusterName);
+ clusters.getCluster(clusterName)
+ .setDesiredStackVersion(new StackId("HDP-0.1"));
+ String serviceName1 = "HDFS";
+ String serviceName2 = "MAPREDUCE";
+ String componentName1 = "NAMENODE";
+ String componentName2 = "DATANODE";
+ String componentName3 = "HDFS_CLIENT";
+ String componentName4 = "JOBTRACKER";
+ String componentName5 = "TASKTRACKER";
+ String componentName6 = "MAPREDUCE_CLIENT";
+ Map<String, String> mapRequestProps = new HashMap<String, String>();
+ mapRequestProps.put("context", "Called from a test");
+
+ createService(clusterName, serviceName1, null);
+ createService(clusterName, serviceName2, null);
+
+ createServiceComponent(clusterName, serviceName1, componentName1,
+ State.INIT);
+ createServiceComponent(clusterName, serviceName1, componentName2,
+ State.INIT);
+ createServiceComponent(clusterName, serviceName1, componentName3,
+ State.INIT);
+ createServiceComponent(clusterName, serviceName2, componentName4,
+ State.INIT);
+ createServiceComponent(clusterName, serviceName2, componentName5,
+ State.INIT);
+ createServiceComponent(clusterName, serviceName2, componentName6,
+ State.INIT);
+
+ String host1 = "h1";
+ String host2 = "h2";
+ String host3 = "h3";
+ clusters.addHost(host1);
+ clusters.addHost(host2);
+ clusters.addHost(host3);
+ clusters.getHost("h1").setOsType("centos6");
+ clusters.getHost("h1").persist();
+ clusters.getHost("h2").setOsType("centos6");
+ clusters.getHost("h2").persist();
+ clusters.getHost("h3").setOsType("centos6");
+ clusters.getHost("h3").persist();
+
+ clusters.mapHostToCluster(host1, clusterName);
+ clusters.mapHostToCluster(host2, clusterName);
+ clusters.mapHostToCluster(host3, clusterName);
+
+ createServiceComponentHost(clusterName, serviceName1, componentName1,
+ host1, null);
+ createServiceComponentHost(clusterName, serviceName1, componentName2,
+ host1, null);
+ createServiceComponentHost(clusterName, serviceName2, componentName4,
+ host1, null);
+ createServiceComponentHost(clusterName, serviceName2, componentName5,
+ host1, null);
+ createServiceComponentHost(clusterName, serviceName1, componentName2,
+ host2, null);
+ createServiceComponentHost(clusterName, serviceName1, componentName3,
+ host2, null);
+ createServiceComponentHost(clusterName, serviceName2, componentName6,
+ host2, null);
+ createServiceComponentHost(clusterName, serviceName1, componentName3,
+ host3, null);
+
+ // Create and attach config
+ Map<String, String> configs = new HashMap<String, String>();
+ configs.put("a", "b");
+
+ ConfigurationRequest cr1,cr2,cr3;
+ cr1 = new ConfigurationRequest(clusterName, "core-site","version1",
+ configs);
+ cr2 = new ConfigurationRequest(clusterName, "hdfs-site","version1",
+ configs);
+
+ ClusterRequest crReq = new ClusterRequest(null, clusterName, null, null);
+ crReq.setDesiredConfig(cr1);
+ controller.updateCluster(crReq, null);
+ crReq = new ClusterRequest(null, clusterName, null, null);
+ crReq.setDesiredConfig(cr2);
+ controller.updateCluster(crReq, null);
+
+ // Install
+ long requestId1 = installService(clusterName, serviceName1);
+
+ List<Stage> stages = actionDB.getAllStages(requestId1);
+ Assert.assertEquals(2, stages.get(0).getOrderedHostRoleCommands().get(0)
+ .getExecutionCommandWrapper().getExecutionCommand()
+ .getConfigurationTags().size());
+
+ installService(clusterName, serviceName2);
+
+ // Start
+ startService(clusterName, serviceName1);
+ startService(clusterName, serviceName2);
+
+ // Reconfigure
+ configs.clear();
+ configs.put("c", "d");
+ cr3 = new ConfigurationRequest(clusterName, "core-site","version122",
+ configs);
+ crReq = new ClusterRequest(null, clusterName, null, null);
+ crReq.setDesiredConfig(cr3);
+ controller.updateCluster(crReq, null);
+
+ // Stop HDFS & MAPREDUCE
+ stopService(clusterName, serviceName1);
+ stopService(clusterName, serviceName2);
+
+ // Start
+ long requestId2 = startService(clusterName, serviceName1);
+ long requestId3 = startService(clusterName, serviceName2);
+
+ stages = actionDB.getAllStages(requestId2);
+ stages.addAll(actionDB.getAllStages(requestId3));
+ HostRoleCommand hdfsCmd = null;
+ HostRoleCommand mapRedCmd = null;
+ for (Stage stage : stages) {
+ List<HostRoleCommand> hrcs = stage.getOrderedHostRoleCommands();
+
+ for (HostRoleCommand hrc : hrcs) {
+ LOG.debug("role: " + hrc.getRole());
+ if (hrc.getRole().toString().equals("HDFS_CLIENT"))
+ hdfsCmd = hrc;
+ if (hrc.getRole().toString().equals("MAPREDUCE_CLIENT"))
+ mapRedCmd = hrc;
+ }
+ }
+ Assert.assertNotNull(hdfsCmd);
+ ExecutionCommand execCmd = hdfsCmd.getExecutionCommandWrapper()
+ .getExecutionCommand();
+ Assert.assertEquals(2, execCmd.getConfigurationTags().size());
+ Assert.assertEquals("version122", execCmd.getConfigurationTags().get
+ ("core-site").get("tag"));
+ Assert.assertEquals("d", execCmd.getConfigurations().get("core-site")
+ .get("c"));
+ // Check if MapReduce client is reinstalled
+ Assert.assertNotNull(mapRedCmd);
+ }
+
+ @Test
public void testClientServiceSmokeTests() throws AmbariException {
String clusterName = "foo1";
createCluster(clusterName);