You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ab...@apache.org on 2014/01/30 21:20:34 UTC
git commit: AMBARI-4465 - Update component name mapping in
ClusterDefinition
Updated Branches:
refs/heads/trunk e8d045af2 -> c4a778da0
AMBARI-4465 - Update component name mapping in ClusterDefinition
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/c4a778da
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/c4a778da
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/c4a778da
Branch: refs/heads/trunk
Commit: c4a778da041ee56d01ef2dc88fd7fcfebe2a0680
Parents: e8d045a
Author: Artem Baranchuk <ab...@hortonworks.com>
Authored: Wed Jan 29 21:39:00 2014 +0200
Committer: Artem Baranchuk <ab...@hortonworks.com>
Committed: Thu Jan 30 22:19:05 2014 +0200
----------------------------------------------------------------------
.../apache/ambari/msi/ClusterDefinition.java | 86 +++++++++++++++-----
.../apache/ambari/scom/SQLProviderModule.java | 3 +
.../ambari/msi/ClusterDefinitionTest.java | 24 ++++++
.../scom/TestClusterDefinitionProvider.java | 6 ++
.../test/resources/clusterproperties_HDP2.txt | 49 +++++++++++
5 files changed, 149 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java
----------------------------------------------------------------------
diff --git a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java
index 3e95ebd..a2da723 100644
--- a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java
+++ b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/msi/ClusterDefinition.java
@@ -22,6 +22,7 @@ import org.apache.ambari.scom.ClusterDefinitionProvider;
import org.apache.ambari.scom.HostInfoProvider;
import org.apache.ambari.server.controller.internal.ResourceImpl;
import org.apache.ambari.server.controller.spi.Resource;
+import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader;
import java.io.IOException;
@@ -39,7 +40,7 @@ import java.util.Set;
public class ClusterDefinition {
private static final String HEADER_TAG = "#";
- private static final String HOSTS_HEADER = "Hosts";
+ private static final String HOSTS_HEADER = "hosts";
private final Set<String> services = new HashSet<String>();
private final Set<String> hosts = new HashSet<String>();
@@ -64,9 +65,9 @@ public class ClusterDefinition {
* Component name mapping to account for differences in what is provided by the MSI
* and what is expected by the Ambari providers.
*/
- private static final Map<String, Set<String>> componentNameMap = new HashMap<String, Set<String>>();
+ private final Map<String, Set<String>> componentNameMap = new HashMap<String, Set<String>>();
- static {
+ private void initComponentNameMap() {
componentNameMap.put("NAMENODE_HOST", Collections.singleton("NAMENODE"));
componentNameMap.put("SECONDARY_NAMENODE_HOST", Collections.singleton("SECONDARY_NAMENODE"));
componentNameMap.put("OOZIE_SERVER_HOST", Collections.singleton("OOZIE_SERVER"));
@@ -76,15 +77,8 @@ public class ClusterDefinition {
componentNameMap.put("HBASE_REGIONSERVERS", Collections.singleton("HBASE_REGIONSERVER"));
componentNameMap.put("ZOOKEEPER_HOSTS", Collections.singleton("ZOOKEEPER_SERVER"));
- Set<String> mapReduceComponents = new HashSet<String>();
- mapReduceComponents.add("JOBTRACKER");
- mapReduceComponents.add("HISTORY_SERVER");
- componentNameMap.put("JOBTRACKER_HOST", mapReduceComponents);
-
Set<String> slaveComponents = new HashSet<String>();
slaveComponents.add("DATANODE");
- slaveComponents.add("TASKTRACKER");
-
componentNameMap.put("SLAVE_HOSTS", slaveComponents);
Set<String> hiveComponents = new HashSet<String>();
@@ -92,22 +86,44 @@ public class ClusterDefinition {
hiveComponents.add("HIVE_SERVER2");
hiveComponents.add("HIVE_METASTORE");
hiveComponents.add("HIVE_CLIENT");
-
componentNameMap.put("HIVE_SERVER_HOST", hiveComponents);
+
+ Integer majorStackVersion = getMajorStackVersion();
+ if(majorStackVersion != null) {
+ if(majorStackVersion == 1) {
+ Set<String> mapReduceComponents = new HashSet<String>();
+ mapReduceComponents.add("JOBTRACKER");
+ mapReduceComponents.add("HISTORY_SERVER");
+ componentNameMap.put("JOBTRACKER_HOST", mapReduceComponents);
+
+ slaveComponents.add("TASKTRACKER");
+ }
+ if(majorStackVersion == 2) {
+ componentNameMap.put("JOURNALNODE_HOST", Collections.singleton("JOURNALNODE"));
+
+ Set<String> mapReduce2Components = new HashSet<String>();
+ mapReduce2Components.add("HISTORY_SERVER");
+ mapReduce2Components.add("RESOURCEMANAGER");
+ componentNameMap.put("RESOURCEMANAGER_HOST", mapReduce2Components);
+
+ slaveComponents.add("NODEMANAGER");
+ //hiveComponents.add("MYSQL_SERVER");
+
+ Set<String> clientHosts = new HashSet<String>();
+ componentNameMap.put("CLIENT_HOSTS", clientHosts);
+ }
+ }
}
/**
* Component service mapping .
*/
- private static final Map<String, String> componentServiceMap = new HashMap<String, String>();
+ private final Map<String, String> componentServiceMap = new HashMap<String, String>();
- static {
+ private void initComponentServiceMap() {
componentServiceMap.put("NAMENODE", "HDFS");
componentServiceMap.put("DATANODE", "HDFS");
componentServiceMap.put("SECONDARY_NAMENODE", "HDFS");
- componentServiceMap.put("JOBTRACKER", "MAPREDUCE");
- componentServiceMap.put("HISTORY_SERVER", "MAPREDUCE");
- componentServiceMap.put("TASKTRACKER", "MAPREDUCE");
componentServiceMap.put("HIVE_SERVER", "HIVE");
componentServiceMap.put("HIVE_SERVER2", "HIVE");
componentServiceMap.put("HIVE_METASTORE", "HIVE");
@@ -118,8 +134,23 @@ public class ClusterDefinition {
componentServiceMap.put("HBASE_MASTER", "HBASE");
componentServiceMap.put("HBASE_REGIONSERVER", "HBASE");
componentServiceMap.put("ZOOKEEPER_SERVER", "ZOOKEEPER");
- }
+ Integer majorStackVersion = getMajorStackVersion();
+ if(majorStackVersion != null) {
+ if(majorStackVersion == 1) {
+ componentServiceMap.put("JOBTRACKER", "MAPREDUCE");
+ componentServiceMap.put("HISTORY_SERVER", "MAPREDUCE");
+ componentServiceMap.put("TASKTRACKER", "MAPREDUCE");
+ }
+ if(majorStackVersion == 2) {
+ componentServiceMap.put("HISTORY_SERVER", "MAPREDUCE2");
+ componentServiceMap.put("JOURNALNODE", "HDFS");
+ componentServiceMap.put("NODEMANAGER", "YARN");
+ componentServiceMap.put("RESOURCEMANAGER", "YARN");
+ //componentServiceMap.put("MYSQL_SERVER", "HIVE");
+ }
+ }
+ }
// ----- Constructors ------------------------------------------------------
@@ -137,6 +168,8 @@ public class ClusterDefinition {
this.clusterName = definitionProvider.getClusterName();
this.versionId = definitionProvider.getVersionId();
+ init();
+
try {
readClusterDefinition();
} catch (IOException e) {
@@ -145,6 +178,21 @@ public class ClusterDefinition {
}
}
+ private void init() {
+ initComponentNameMap();
+ initComponentServiceMap();
+ }
+
+ private Integer getMajorStackVersion() {
+ if(StringUtils.isNotEmpty(versionId)) {
+ String majorVersion = StringUtils.substring(versionId, 4, 5);
+ if(StringUtils.isNotEmpty(majorVersion)) {
+ return Integer.parseInt(majorVersion);
+ }
+ }
+ return null;
+ }
+
// ----- ClusterDefinition -------------------------------------------------
/**
@@ -481,8 +529,8 @@ public class ClusterDefinition {
if (line.startsWith(HEADER_TAG)) {
- String header = line.substring(HEADER_TAG.length());
- hostsSection = header.equals(HOSTS_HEADER);
+ String header = line.substring(HEADER_TAG.length()).toLowerCase();
+ hostsSection = header.equalsIgnoreCase(HOSTS_HEADER);
if (!hostsSection && (header.startsWith(HOSTS_HEADER) ) ){
char c = header.charAt(HOSTS_HEADER.length());
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java
----------------------------------------------------------------------
diff --git a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java
index 439ac00..cbf2bfe 100644
--- a/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java
+++ b/contrib/ambari-scom/ambari-scom-server/src/main/java/org/apache/ambari/scom/SQLProviderModule.java
@@ -76,6 +76,9 @@ public class SQLProviderModule extends DefaultProviderModule implements HostInfo
serviceNames.put("DATANODE", "datanode");
serviceNames.put("TASKTRACKER", "tasktracker");
serviceNames.put("WEBHCAT_SERVER", "templeton");
+ serviceNames.put("NODEMANAGER", "nodemanager");
+ serviceNames.put("RESOURCEMANAGER", "resourcemanager");
+ serviceNames.put("JOURNALNODE", "journalnode");
}
private static final String STATE_PREFIX = "STATE : ";
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java
----------------------------------------------------------------------
diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java
index c048df2..0652ea6 100644
--- a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java
+++ b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/msi/ClusterDefinitionTest.java
@@ -283,4 +283,28 @@ public class ClusterDefinitionTest {
verify(mockClusterDefinitionProvider, mockHostInfoProvider, mockStateProvider);
}
+
+ @Test
+ public void testHDP2ServicesAndComponents() {
+ StateProvider mockStateProvider = createStrictMock(StateProvider.class);
+ ClusterDefinitionProvider mockClusterDefinitionProvider = createStrictMock(ClusterDefinitionProvider.class);
+ HostInfoProvider mockHostInfoProvider = createStrictMock(HostInfoProvider.class);
+
+ TestClusterDefinitionProvider testClusterDefinitionProvider = new TestClusterDefinitionProvider("clusterproperties_HDP2.txt", "myCluster", "HDP-2.0.6");
+
+ expect(mockClusterDefinitionProvider.getClusterName()).andDelegateTo(testClusterDefinitionProvider);
+ expect(mockClusterDefinitionProvider.getVersionId()).andDelegateTo(testClusterDefinitionProvider);
+ expect(mockClusterDefinitionProvider.getInputStream()).andDelegateTo(testClusterDefinitionProvider);
+
+ replay(mockClusterDefinitionProvider, mockHostInfoProvider, mockStateProvider);
+
+ ClusterDefinition clusterDefinition = new ClusterDefinition(mockStateProvider, mockClusterDefinitionProvider, mockHostInfoProvider);
+ Assert.assertFalse(clusterDefinition.getServices().contains("MAPREDUCE"));
+ Assert.assertTrue(clusterDefinition.getServices().contains("YARN"));
+ Assert.assertTrue(clusterDefinition.getServices().contains("MAPREDUCE2"));
+ Assert.assertTrue(clusterDefinition.getComponents("YARN").contains("NODEMANAGER"));
+ Assert.assertTrue(clusterDefinition.getComponents("YARN").contains("RESOURCEMANAGER"));
+
+ verify(mockClusterDefinitionProvider, mockHostInfoProvider, mockStateProvider);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java
----------------------------------------------------------------------
diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java
index 98c6869..58e057b 100644
--- a/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java
+++ b/contrib/ambari-scom/ambari-scom-server/src/test/java/org/apache/ambari/scom/TestClusterDefinitionProvider.java
@@ -27,4 +27,10 @@ public class TestClusterDefinitionProvider extends ClusterDefinitionProvider {
setClusterName("myCluster");
setVersionId("HDP-1.2.9");
}
+
+ public TestClusterDefinitionProvider (String fileName, String clusterName, String versionId) {
+ setFileName(fileName);
+ setClusterName(clusterName);
+ setVersionId(versionId);
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/c4a778da/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt
----------------------------------------------------------------------
diff --git a/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt
new file mode 100644
index 0000000..e359aa9
--- /dev/null
+++ b/contrib/ambari-scom/ambari-scom-server/src/test/resources/clusterproperties_HDP2.txt
@@ -0,0 +1,49 @@
+### Licensed to the Apache Software Foundation (ASF) under one or more
+### contributor license agreements. See the NOTICE file distributed with
+### this work for additional information regarding copyright ownership.
+### The ASF licenses this file to You under the Apache License, Version 2.0
+### (the "License"); you may not use this file except in compliance with
+### the License. You may obtain a copy of the License at
+###
+### http://www.apache.org/licenses/LICENSE-2.0
+###
+### Unless required by applicable law or agreed to in writing, software
+### distributed under the License is distributed on an "AS IS" BASIS,
+### WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+### See the License for the specific language governing permissions and
+### limitations under the License.
+
+#Log directory
+HDP_LOG_DIR=c:\hadoop\logs
+
+#Data directory
+HDP_DATA_DIR=c:\hadoop\data
+
+#hosts
+NAMENODE_HOST=tbeerb-ascom21
+SECONDARY_NAMENODE_HOST=tbeerb-ascom21
+RESOURCEMANAGER_HOST=tbeerb-ascom21
+HIVE_SERVER_HOST=tbeerb-ascom21
+OOZIE_SERVER_HOST=tbeerb-ascom21
+WEBHCAT_HOST=tbeerb-ascom21
+SLAVE_HOSTS=tbeerb-ascom21
+CLIENT_HOSTS=tbeerb-ascom21
+HBASE_MASTER=tbeerb-ascom21
+HBASE_REGIONSERVERS=tbeerb-ascom21
+ZOOKEEPER_HOSTS=tbeerb-ascom21
+FLUME_HOSTS=tbeerb-ascom21
+
+#Database host
+DB_FLAVOR=derby
+DB_HOSTNAME=tbeerb-ascom21
+DB_PORT=1527
+
+#Hive properties
+HIVE_DB_NAME=hive
+HIVE_DB_USERNAME=hive
+HIVE_DB_PASSWORD=hive
+
+#Oozie properties
+OOZIE_DB_NAME=oozie
+OOZIE_DB_USERNAME=oozie
+OOZIE_DB_PASSWORD=oozie