You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/02/19 14:26:27 UTC
[10/21] ambari git commit: AMBARI-15064. RU/EU can't start if hosts
have name in MixedCASE in configs for NameNode, HBASE Master,
ResourceManager (alejandro)
AMBARI-15064. RU/EU can't start if hosts have name in MixedCASE in configs for NameNode, HBASE Master, ResourceManager (alejandro)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/8f58e5f9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/8f58e5f9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/8f58e5f9
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 8f58e5f97e0e6c7853956ad4f48d45603cee9d7f
Parents: 7da4706
Author: Alejandro Fernandez <af...@hortonworks.com>
Authored: Thu Feb 18 11:21:04 2016 -0800
Committer: Alejandro Fernandez <af...@hortonworks.com>
Committed: Thu Feb 18 11:21:16 2016 -0800
----------------------------------------------------------------------
.../controller/AmbariActionExecutionHelper.java | 13 +-
.../internal/UpgradeResourceProvider.java | 1 +
.../ambari/server/stack/MasterHostResolver.java | 53 +++++--
.../AmbariManagementControllerTest.java | 2 +-
.../ambari/server/state/UpgradeHelperTest.java | 140 +++++++++++++++++--
5 files changed, 178 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
index 795dfa7..88180c0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/AmbariActionExecutionHelper.java
@@ -262,6 +262,13 @@ public class AmbariActionExecutionHelper {
final String serviceName = actionContext.getExpectedServiceName();
final String componentName = actionContext.getExpectedComponentName();
+ LOG.debug("Called addExecutionCommandsToStage() for serviceName: {}, componentName: {}.", serviceName, componentName);
+ if (resourceFilter.getHostNames().isEmpty()) {
+ LOG.debug("Resource filter has no hostnames.");
+ } else {
+ LOG.debug("Resource filter has hosts: {}", StringUtils.join(resourceFilter.getHostNames(), ", "));
+ }
+
if (null != cluster) {
StackId stackId = cluster.getCurrentStackVersion();
if (serviceName != null && !serviceName.isEmpty()) {
@@ -275,6 +282,7 @@ public class AmbariActionExecutionHelper {
stackId.getStackVersion(), serviceName, componentName);
} catch (ObjectNotFoundException e) {
// do nothing, componentId is checked for null later
+ LOG.error("Did not find service {} and component {} in stack {}.", serviceName, componentName, stackId.getStackName());
}
} else {
for (String component : cluster.getService(serviceName).getServiceComponents().keySet()) {
@@ -288,6 +296,7 @@ public class AmbariActionExecutionHelper {
// All hosts are valid target host
candidateHosts.addAll(clusters.getHostsForCluster(cluster.getClusterName()).keySet());
}
+ LOG.debug("Request for service {} and component {} is set to run on candidate hosts: {}.", serviceName, componentName, StringUtils.join(candidateHosts, ", "));
// Filter hosts that are in MS
Set<String> ignoredHosts = maintenanceStateHelper.filterHostsInMaintenanceState(
@@ -301,7 +310,9 @@ public class AmbariActionExecutionHelper {
}
}
);
+
if (! ignoredHosts.isEmpty()) {
+ LOG.debug("Hosts to ignore: {}.", StringUtils.join(ignoredHosts, ", "));
LOG.debug("Ignoring action for hosts due to maintenance state." +
"Ignored hosts =" + ignoredHosts + ", component="
+ componentName + ", service=" + serviceName
@@ -323,7 +334,7 @@ public class AmbariActionExecutionHelper {
for (String hostname : resourceFilter.getHostNames()) {
if (!candidateHosts.contains(hostname)) {
throw new AmbariException("Request specifies host " + hostname +
- " but its not a valid host based on the " +
+ " but it is not a valid host based on the " +
"target service=" + serviceName + " and component=" + componentName);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index 70440fc..e5664c2 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1175,6 +1175,7 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
RequestResourceFilter filter = new RequestResourceFilter("", "",
new ArrayList<String>(wrapper.getHosts()));
+ LOG.debug("Analyzing upgrade item {} with tasks: {}.", entity.getText(), entity.getTasks());
Map<String, String> params = getNewParameterMap();
params.put(COMMAND_PARAM_TASKS, entity.getTasks());
params.put(COMMAND_PARAM_VERSION, context.getVersion());
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
index 561350b..360f2b8 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/MasterHostResolver.java
@@ -63,7 +63,7 @@ public class MasterHostResolver {
/**
* Union of status for several services.
*/
- enum Status {
+ protected enum Status {
ACTIVE,
STANDBY
}
@@ -247,11 +247,11 @@ public class MasterHostResolver {
return false;
}
-
/**
* Get mapping of the HDFS Namenodes from the state ("active" or "standby") to the hostname.
* @return Returns a map from the state ("active" or "standby" to the hostname with that state if exactly
* one active and one standby host were found, otherwise, return null.
+ * The hostnames are returned in lowercase.
*/
private Map<Status, String> getNameNodePair() {
Map<Status, String> stateToHost = new HashMap<Status, String>();
@@ -291,7 +291,9 @@ public class MasterHostResolver {
if (null != state && (state.equalsIgnoreCase(Status.ACTIVE.toString()) || state.equalsIgnoreCase(Status.STANDBY.toString()))) {
Status status = Status.valueOf(state.toUpperCase());
- stateToHost.put(status, hp.host);
+ stateToHost.put(status, hp.host.toLowerCase());
+ } else {
+ LOG.error(String.format("Could not retrieve state for NameNode %s from property %s by querying JMX.", hp.host, key));
}
} catch (MalformedURLException e) {
LOG.error(e.getMessage());
@@ -304,6 +306,12 @@ public class MasterHostResolver {
return null;
}
+ /**
+ * Resolve the name of the Resource Manager master and convert the hostname to lowercase.
+ * @param cluster Cluster
+ * @param hostType RM hosts
+ * @throws MalformedURLException
+ */
private void resolveResourceManagers(Cluster cluster, HostsType hostType) throws MalformedURLException {
LinkedHashSet<String> orderedHosts = new LinkedHashSet<String>(hostType.hosts);
@@ -320,18 +328,24 @@ public class MasterHostResolver {
if (null != value) {
if (null == hostType.master) {
- hostType.master = hostname;
+ hostType.master = hostname.toLowerCase();
}
// Quick and dirty to make sure the master is last in the list
- orderedHosts.remove(hostname);
- orderedHosts.add(hostname);
+ orderedHosts.remove(hostname.toLowerCase());
+ orderedHosts.add(hostname.toLowerCase());
}
}
hostType.hosts = orderedHosts;
}
+ /**
+ * Resolve the HBASE master and convert the hostname to lowercase.
+ * @param cluster Cluster
+ * @param hostsType HBASE master host.
+ * @throws AmbariException
+ */
private void resolveHBaseMasters(Cluster cluster, HostsType hostsType) throws AmbariException {
String hbaseMasterInfoPortProperty = "hbase.master.info.port";
String hbaseMasterInfoPortValue = m_configHelper.getValueFromDesiredConfigurations(cluster, ConfigHelper.HBASE_SITE, hbaseMasterInfoPortProperty);
@@ -348,22 +362,31 @@ public class MasterHostResolver {
if (null != value) {
Boolean bool = Boolean.valueOf(value);
if (bool.booleanValue()) {
- hostsType.master = hostname;
+ hostsType.master = hostname.toLowerCase();
} else {
- hostsType.secondary = hostname;
+ hostsType.secondary = hostname.toLowerCase();
}
}
-
}
}
- private String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
- boolean asQuery) {
+ protected String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
+ boolean asQuery) {
return queryJmxBeanValue(hostname, port, beanName, attributeName, asQuery, false);
}
- private String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
- boolean asQuery, boolean encrypted) {
+ /**
+ * Query the JMX attribute at http(s)://$server:$port/jmx?qry=$query or http(s)://$server:$port/jmx?get=$bean::$attribute
+ * @param hostname host name
+ * @param port port number
+ * @param beanName if asQuery is false, then search for this bean name
+ * @param attributeName if asQuery is false, then search for this attribute name
+ * @param asQuery whether to search bean or query
+ * @param encrypted true if using https instead of http.
+ * @return The jmx value.
+ */
+ protected String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
+ boolean asQuery, boolean encrypted) {
String protocol = encrypted ? "https://" : "http://";
String endPoint = protocol + (asQuery ?
@@ -385,9 +408,9 @@ public class MasterHostResolver {
return jmxBeans.get("beans").get(0).get(attributeName);
} catch (Exception e) {
if (LOG.isDebugEnabled()) {
- LOG.info("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname, e);
+ LOG.debug("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname, e);
} else {
- LOG.info("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname);
+ LOG.debug("Could not load JMX from {}/{} from {}", beanName, attributeName, hostname);
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index a0ac966..ffee3fa 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -4650,7 +4650,7 @@ public class AmbariManagementControllerTest {
actionRequest = new ExecuteActionRequest("c1", null, "a2", resourceFilters, null, params, false);
expectActionCreationErrorWithMessage(actionRequest, requestProperties,
- "Request specifies host h6 but its not a valid host based on the target service=HDFS and component=DATANODE");
+ "Request specifies host h6 but it is not a valid host based on the target service=HDFS and component=DATANODE");
hosts.clear();
hosts.add("h1");
http://git-wip-us.apache.org/repos/asf/ambari/blob/8f58e5f9/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
index eb5bf62..4ea8f15 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/state/UpgradeHelperTest.java
@@ -95,21 +95,23 @@ public class UpgradeHelperTest {
private AmbariManagementController m_managementController;
private Gson m_gson = new Gson();
- @Before
- public void before() throws Exception {
+ /**
+ * Because test cases need to share config mocks, put common ones in this function.
+ * @throws Exception
+ */
+ private void setConfigMocks() throws Exception {
// configure the mock to return data given a specific placeholder
m_configHelper = EasyMock.createNiceMock(ConfigHelper.class);
+ expect(m_configHelper.getPlaceholderValueFromDesiredConfigurations(
+ EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn("placeholder-rendered-properly").anyTimes();
+ expect(m_configHelper.getEffectiveDesiredTags(
+ EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))).andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
+ }
- expect(
- m_configHelper.getPlaceholderValueFromDesiredConfigurations(
- EasyMock.anyObject(Cluster.class), EasyMock.eq("{{foo/bar}}"))).andReturn(
- "placeholder-rendered-properly").anyTimes();
-
- expect(
- m_configHelper.getEffectiveDesiredTags(
- EasyMock.anyObject(Cluster.class), EasyMock.anyObject(String.class))).
- andReturn(new HashMap<String, Map<String, String>>()).anyTimes();
-
+ @Before
+ public void before() throws Exception {
+ setConfigMocks();
+ // Most test cases can replay the common config mocks. If any test case needs custom ones, it can re-initialize m_configHelper;
replay(m_configHelper);
final InMemoryDefaultTestModule injectorModule = new InMemoryDefaultTestModule() {
@@ -1244,7 +1246,7 @@ public class UpgradeHelperTest {
Service s = c.getService("ZOOKEEPER");
ServiceComponent sc = s.addServiceComponent("ZOOKEEPER_SERVER");
- ServiceComponentHost sch1 =sc.addServiceComponentHost("h1");
+ ServiceComponentHost sch1 = sc.addServiceComponentHost("h1");
sch1.setVersion("2.1.1.0-1234");
ServiceComponentHost sch2 = sc.addServiceComponentHost("h2");
@@ -1252,7 +1254,6 @@ public class UpgradeHelperTest {
List<ServiceComponentHost> schs = c.getServiceComponentHosts("ZOOKEEPER", "ZOOKEEPER_SERVER");
assertEquals(2, schs.size());
-
MasterHostResolver mhr = new MasterHostResolver(null, c, "2.1.1.0-1234");
HostsType ht = mhr.getMasterAndHosts("ZOOKEEPER", "ZOOKEEPER_SERVER");
@@ -1267,6 +1268,117 @@ public class UpgradeHelperTest {
assertEquals("h2", ht.hosts.iterator().next());
}
+ /**
+ * Test that MasterHostResolver is case-insensitive even if configs have hosts in upper case for NameNode.
+ * @throws Exception
+ */
+ @Test
+ public void testResolverCaseInsensitive() throws Exception {
+ Clusters clusters = injector.getInstance(Clusters.class);
+ ServiceFactory serviceFactory = injector.getInstance(ServiceFactory.class);
+
+ String clusterName = "c1";
+ String version = "2.1.1.0-1234";
+
+ StackId stackId = new StackId("HDP-2.1.1");
+ clusters.addCluster(clusterName, stackId);
+ Cluster c = clusters.getCluster(clusterName);
+
+ helper.getOrCreateRepositoryVersion(stackId,
+ c.getDesiredStackVersion().getStackVersion());
+
+ c.createClusterVersion(stackId,
+ c.getDesiredStackVersion().getStackVersion(), "admin",
+ RepositoryVersionState.UPGRADING);
+
+ for (int i = 0; i < 2; i++) {
+ String hostName = "h" + (i+1);
+ clusters.addHost(hostName);
+ Host host = clusters.getHost(hostName);
+
+ Map<String, String> hostAttributes = new HashMap<String, String>();
+ hostAttributes.put("os_family", "redhat");
+ hostAttributes.put("os_release_version", "6");
+
+ host.setHostAttributes(hostAttributes);
+
+ host.persist();
+ clusters.mapHostToCluster(hostName, clusterName);
+ }
+
+ // Add services
+ c.addService(serviceFactory.createNew(c, "HDFS"));
+
+ Service s = c.getService("HDFS");
+ ServiceComponent sc = s.addServiceComponent("NAMENODE");
+ sc.addServiceComponentHost("h1");
+ sc.addServiceComponentHost("h2");
+
+ List<ServiceComponentHost> schs = c.getServiceComponentHosts("HDFS", "NAMENODE");
+ assertEquals(2, schs.size());
+
+ setConfigMocks();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.nameservices")).andReturn("ha").anyTimes();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.ha.namenodes.ha")).andReturn("nn1,nn2").anyTimes();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.http.policy")).andReturn("HTTP_ONLY").anyTimes();
+
+ // Notice that these names are all caps.
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn1")).andReturn("H1:50070").anyTimes();
+ expect(m_configHelper.getValueFromDesiredConfigurations(c, "hdfs-site", "dfs.namenode.http-address.ha.nn2")).andReturn("H2:50070").anyTimes();
+ replay(m_configHelper);
+
+ MasterHostResolver mhr = new MockMasterHostResolver(m_configHelper, c, version);
+
+ HostsType ht = mhr.getMasterAndHosts("HDFS", "NAMENODE");
+ assertNotNull(ht.master);
+ assertNotNull(ht.secondary);
+ assertEquals(2, ht.hosts.size());
+
+ // Should be stored in lowercase.
+ assertTrue(ht.hosts.contains("h1"));
+ assertTrue(ht.hosts.contains("h1"));
+ }
+
+ /**
+ * Extend {@link org.apache.ambari.server.stack.MasterHostResolver} in order to overwrite the JMX methods.
+ */
+ private class MockMasterHostResolver extends MasterHostResolver {
+
+ public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster) {
+ this(configHelper, cluster, null);
+ }
+
+ public MockMasterHostResolver(ConfigHelper configHelper, Cluster cluster, String version) {
+ super(configHelper, cluster, version);
+ }
+
+ /**
+ * Mock the call to get JMX Values.
+ * @param hostname host name
+ * @param port port number
+ * @param beanName if asQuery is false, then search for this bean name
+ * @param attributeName if asQuery is false, then search for this attribute name
+ * @param asQuery whether to search bean or query
+ * @param encrypted true if using https instead of http.
+ * @return
+ */
+ @Override
+ public String queryJmxBeanValue(String hostname, int port, String beanName, String attributeName,
+ boolean asQuery, boolean encrypted) {
+
+ if (beanName.equalsIgnoreCase("Hadoop:service=NameNode,name=NameNodeStatus") && attributeName.equalsIgnoreCase("State") && asQuery) {
+ switch (hostname) {
+ case "H1":
+ return Status.ACTIVE.toString();
+ case "H2":
+ return Status.STANDBY.toString();
+ default:
+ return "UNKNOWN_NAMENODE_STATUS_FOR_THIS_HOST";
+ }
+ }
+ return "NOT_MOCKED";
+ }
+ }
private class MockModule implements Module {