You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sr...@apache.org on 2014/08/28 03:04:58 UTC
git commit: AMBARI-7043. BE: Move stack_advisor.py abstract class one
folder up - RPM and DEB package changes.
Repository: ambari
Updated Branches:
refs/heads/trunk 39c7ae74b -> d87cf7f8d
AMBARI-7043. BE: Move stack_advisor.py abstract class one folder up -
RPM and DEB package changes.
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d87cf7f8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d87cf7f8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d87cf7f8
Branch: refs/heads/trunk
Commit: d87cf7f8d0fee68210296085954a7a1e6ce484c4
Parents: 39c7ae7
Author: Srimanth Gunturi <sg...@hortonworks.com>
Authored: Wed Aug 27 10:56:25 2014 -0700
Committer: Srimanth Gunturi <sg...@hortonworks.com>
Committed: Wed Aug 27 18:04:00 2014 -0700
----------------------------------------------------------------------
ambari-server/pom.xml | 22 +++++
.../src/main/resources/scripts/stack_advisor.py | 2 +-
.../main/resources/stacks/HDP/stack_advisor.py | 37 --------
.../src/main/resources/stacks/stack_advisor.py | 37 ++++++++
.../server/api/services/AmbariMetaInfoTest.java | 4 +-
.../AmbariManagementControllerTest.java | 2 +-
.../src/test/python/TestStackAdvisor.py | 93 ++++++++++++++++++++
.../stacks/2.0.6/common/test_stack_advisor.py | 2 +-
.../stacks/XYZ/1.0.0/services/stack_advisor.py | 67 ++++++++++++++
.../stacks/XYZ/1.0.1/services/stack_advisor.py | 30 +++++++
10 files changed, 255 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 2302c62..6279834 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -463,6 +463,17 @@
</sources>
</mapping>
<mapping>
+ <directory>/var/lib/ambari-server/resources/stacks</directory>
+ <filemode>755</filemode>
+ <username>root</username>
+ <groupname>root</groupname>
+ <sources>
+ <source>
+ <location>target/classes/stacks/stack_advisor.py</location>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
<directory>/usr/lib/python2.6/site-packages/ambari_server</directory>
<filemode>755</filemode>
<username>root</username>
@@ -804,6 +815,17 @@
</mapper>
</data>
<data>
+ <src>target/classes/stacks/stack_advisor.py</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <prefix>/var/lib/ambari-server/resources/stacks</prefix>
+ <user>root</user>
+ <group>root</group>
+ <filemode>755</filemode>
+ </mapper>
+ </data>
+ <data>
<src>src/main/python/ambari_server</src>
<type>directory</type>
<mapper>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/main/resources/scripts/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/scripts/stack_advisor.py b/ambari-server/src/main/resources/scripts/stack_advisor.py
index 61d1e09..ed50866 100755
--- a/ambari-server/src/main/resources/scripts/stack_advisor.py
+++ b/ambari-server/src/main/resources/scripts/stack_advisor.py
@@ -33,7 +33,7 @@ ALL_ACTIONS = [ RECOMMEND_COMPONENT_LAYOUT_ACTION, VALIDATE_COMPONENT_LAYOUT_ACT
USAGE = "Usage: <action> <hosts_file> <services_file>\nPossible actions are: {0}\n".format( str(ALL_ACTIONS) )
SCRIPT_DIRECTORY = os.path.dirname(os.path.abspath(__file__))
-STACK_ADVISOR_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, '../stacks/{0}/stack_advisor.py')
+STACK_ADVISOR_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, '../stacks/stack_advisor.py')
STACK_ADVISOR_IMPL_PATH_TEMPLATE = os.path.join(SCRIPT_DIRECTORY, './../stacks/{0}/{1}/services/stack_advisor.py')
STACK_ADVISOR_IMPL_CLASS_TEMPLATE = '{0}{1}StackAdvisor'
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py b/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
deleted file mode 100644
index 5f66fff..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/stack_advisor.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env ambari-python-wrap
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-class StackAdvisor(object):
-
- def recommendComponentLayout(self, services, hosts):
- """Returns Services object with hostnames array populated for components"""
- pass
-
- def validateComponentLayout(self, services, hosts):
- """Returns array of Validation objects about issues with hostnames components assigned to"""
- pass
-
- def recommendConfigurations(self, services, hosts):
- """Returns Services object with configurations object populated"""
- pass
-
- def validateConfigurations(self, services, hosts):
- """Returns array of Validation objects about issues with configuration values provided in services"""
- pass
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/main/resources/stacks/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/stack_advisor.py b/ambari-server/src/main/resources/stacks/stack_advisor.py
new file mode 100644
index 0000000..5f66fff
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/stack_advisor.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+class StackAdvisor(object):
+
+ def recommendComponentLayout(self, services, hosts):
+ """Returns Services object with hostnames array populated for components"""
+ pass
+
+ def validateComponentLayout(self, services, hosts):
+ """Returns array of Validation objects about issues with hostnames components assigned to"""
+ pass
+
+ def recommendConfigurations(self, services, hosts):
+ """Returns Services object with configurations object populated"""
+ pass
+
+ def validateConfigurations(self, services, hosts):
+ """Returns array of Validation objects about issues with configuration values provided in services"""
+ pass
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
index 37d1af4..c08534c 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/api/services/AmbariMetaInfoTest.java
@@ -73,6 +73,7 @@ import com.google.inject.Injector;
public class AmbariMetaInfoTest {
private static final String STACK_NAME_HDP = "HDP";
+ private static final String STACK_NAME_XYZ = "XYZ";
private static final String STACK_VERSION_HDP = "0.1";
private static final String EXT_STACK_NAME = "2.0.6";
private static final String STACK_VERSION_HDP_02 = "0.2";
@@ -88,7 +89,7 @@ public class AmbariMetaInfoTest {
private static final String NON_EXT_VALUE = "XXX";
private static final int REPOS_CNT = 3;
- private static final int STACKS_NAMES_CNT = 1;
+ private static final int STACKS_NAMES_CNT = 2;
private static final int PROPERTIES_CNT = 62;
private static final int OS_CNT = 4;
@@ -408,6 +409,7 @@ public class AmbariMetaInfoTest {
Set<Stack> stackNames = metaInfo.getStackNames();
assertEquals(stackNames.size(), STACKS_NAMES_CNT);
assertTrue(stackNames.contains(new Stack(STACK_NAME_HDP)));
+ assertTrue(stackNames.contains(new Stack(STACK_NAME_XYZ)));
}
@Test
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
index 1bfc25a..5fdd619 100644
--- a/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
+++ b/ambari-server/src/test/java/org/apache/ambari/server/controller/AmbariManagementControllerTest.java
@@ -156,7 +156,7 @@ public class AmbariManagementControllerTest {
private static final String NAGIOS_SERVICE_NAME = "NAGIOS";
private static final int STACK_VERSIONS_CNT = 12;
private static final int REPOS_CNT = 3;
- private static final int STACKS_CNT = 1;
+ private static final int STACKS_CNT = 2;
private static final int STACK_PROPERTIES_CNT = 99;
private static final int STACK_COMPONENTS_CNT = 4;
private static final int OS_CNT = 2;
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/test/python/TestStackAdvisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestStackAdvisor.py b/ambari-server/src/test/python/TestStackAdvisor.py
new file mode 100644
index 0000000..5a1a242
--- /dev/null
+++ b/ambari-server/src/test/python/TestStackAdvisor.py
@@ -0,0 +1,93 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+import os
+
+class TestStackAdvisorInitialization(TestCase):
+
+ def setUp(self):
+ import imp
+
+ self.test_directory = os.path.dirname(os.path.abspath(__file__))
+ stack_advisor_path = os.path.join(self.test_directory, '../../main/resources/scripts/stack_advisor.py')
+ with open(stack_advisor_path, 'rb') as fp:
+ self.stack_advisor = imp.load_module( 'stack_advisor', fp, stack_advisor_path, ('.py', 'rb', imp.PY_SOURCE) )
+
+ def test_stackAdvisorLoadedForNotHDPStack(self):
+ path_template = os.path.join(self.test_directory, '../resources/stacks/{0}/{1}/services/stack_advisor.py')
+ path_template_name = "STACK_ADVISOR_IMPL_PATH_TEMPLATE"
+ setattr(self.stack_advisor, path_template_name, path_template)
+ self.assertEquals(path_template, getattr(self.stack_advisor, path_template_name))
+ instantiate_stack_advisor_method_name = 'instantiateStackAdvisor'
+ instantiate_stack_advisor_method = getattr(self.stack_advisor, instantiate_stack_advisor_method_name)
+ stack_advisor = instantiate_stack_advisor_method("XYZ", "1.0.1", ["1.0.0"])
+ self.assertEquals("XYZ101StackAdvisor", stack_advisor.__class__.__name__)
+ services = {"Versions":
+ {
+ "stack_name":"XYZ",
+ "stack_version":"1.0.1"
+ },
+ "services":[
+ {
+ "StackServices":{
+ "service_name":"YARN"
+ },
+ "components":[
+ {
+ "StackServiceComponents": {
+ "component_name": "RESOURCEMANAGER"
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "APP_TIMELINE_SERVER"
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name":"YARN_CLIENT"
+ }
+ },
+ {
+ "StackServiceComponents": {
+ "component_name": "NODEMANAGER"
+ }
+ }
+ ]
+ }
+ ]
+ }
+ hosts= {
+ "items": [
+ {"Hosts": {"host_name": "host1"}},
+ {"Hosts": {"host_name": "host2"}}
+ ]
+ }
+ config_recommendations = stack_advisor.recommendConfigurations(services, hosts)
+ yarn_configs = config_recommendations["recommendations"]["blueprint"]["configurations"]["yarn-site"]["properties"]
+ '''Check that value is populated from child class, not parent'''
+ self.assertEquals("-Xmx101m", yarn_configs["yarn.nodemanager.resource.memory-mb"])
+
+ def test_stackAdvisorSuperClassIsFoundAndReturnedAsDefaultImpl(self):
+ instantiate_stack_advisor_method_name = 'instantiateStackAdvisor'
+ instantiate_stack_advisor_method = getattr(self.stack_advisor, instantiate_stack_advisor_method_name)
+ '''Not existent stack - to return default implementation'''
+ default_stack_advisor = instantiate_stack_advisor_method("HDP1", "2.0.6", [])
+ self.assertEquals("StackAdvisor", default_stack_advisor.__class__.__name__)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
index b5bdba6..d32fa56 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/common/test_stack_advisor.py
@@ -26,7 +26,7 @@ class TestHDP206StackAdvisor(TestCase):
import os
testDirectory = os.path.dirname(os.path.abspath(__file__))
- stackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/stack_advisor.py')
+ stackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/stack_advisor.py')
hdp206StackAdvisorPath = os.path.join(testDirectory, '../../../../../main/resources/stacks/HDP/2.0.6/services/stack_advisor.py')
hdp206StackAdvisorClassName = 'HDP206StackAdvisor'
with open(stackAdvisorPath, 'rb') as fp:
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/test/resources/stacks/XYZ/1.0.0/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/XYZ/1.0.0/services/stack_advisor.py b/ambari-server/src/test/resources/stacks/XYZ/1.0.0/services/stack_advisor.py
new file mode 100644
index 0000000..ba140bb
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/XYZ/1.0.0/services/stack_advisor.py
@@ -0,0 +1,67 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from stack_advisor import StackAdvisor
+
+class XYZ100StackAdvisor(StackAdvisor):
+
+ def recommendConfigurations(self, services, hosts):
+ stackName = services["Versions"]["stack_name"]
+ stackVersion = services["Versions"]["stack_version"]
+ hostsList = [host["Hosts"]["host_name"] for host in hosts["items"]]
+ servicesList = [service["StackServices"]["service_name"] for service in services["services"]]
+
+ recommendations = {
+ "Versions": {"stack_name": stackName, "stack_version": stackVersion},
+ "hosts": hostsList,
+ "services": servicesList,
+ "recommendations": {
+ "blueprint": {
+ "configurations": {},
+ "host_groups": []
+ },
+ "blueprint_cluster_binding": {
+ "host_groups": []
+ }
+ }
+ }
+
+ configurations = recommendations["recommendations"]["blueprint"]["configurations"]
+ for service in servicesList:
+ calculation = self.recommendServiceConfigurations(service)
+ if calculation is not None:
+ calculation(configurations)
+
+ return recommendations
+
+ def recommendServiceConfigurations(self, service):
+ return {
+ "YARN": self.recommendYARNConfigurations,
+ }.get(service, None)
+
+ def putProperty(self, config, configType):
+ config[configType] = {"properties": {}}
+ def appendProperty(key, value):
+ config[configType]["properties"][key] = str(value)
+ return appendProperty
+
+ def recommendYARNConfigurations(self, configurations):
+ putYarnProperty = self.putProperty(configurations, "yarn-site")
+ putYarnProperty('yarn.nodemanager.resource.memory-mb', "-Xmx100m")
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/d87cf7f8/ambari-server/src/test/resources/stacks/XYZ/1.0.1/services/stack_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/stacks/XYZ/1.0.1/services/stack_advisor.py b/ambari-server/src/test/resources/stacks/XYZ/1.0.1/services/stack_advisor.py
new file mode 100644
index 0000000..74a4b31
--- /dev/null
+++ b/ambari-server/src/test/resources/stacks/XYZ/1.0.1/services/stack_advisor.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import re
+import socket
+import sys
+
+from stack_advisor import StackAdvisor
+
+class XYZ101StackAdvisor(XYZ100StackAdvisor):
+
+ def recommendYARNConfigurations(self, configurations):
+ putYarnProperty = self.putProperty(configurations, "yarn-site")
+ putYarnProperty('yarn.nodemanager.resource.memory-mb', "-Xmx101m")