You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/02/11 10:17:19 UTC
ambari git commit: AMBARI-15007. Make amazon2015 to be part of
redhat6 family (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk af560ae37 -> 6270fe393
AMBARI-15007. Make amazon2015 to be part of redhat6 family (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6270fe39
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6270fe39
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6270fe39
Branch: refs/heads/trunk
Commit: 6270fe39331eb8f8e16a49ad7b8df59edecd2ce5
Parents: af560ae
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Thu Feb 11 11:15:25 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Feb 11 11:15:25 2016 +0200
----------------------------------------------------------------------
.../src/main/python/ambari_commons/os_check.py | 73 +++++++---
.../ambari_commons/resources/os_family.json | 137 +++++++++----------
.../server/state/stack/JsonOsFamilyRoot.java | 38 +++++
.../ambari/server/state/stack/OsFamily.java | 8 +-
.../resources/stacks/HDP/2.2/repos/repoinfo.xml | 12 --
.../resources/stacks/HDP/2.3/repos/repoinfo.xml | 12 --
.../resources/stacks/HDP/2.4/repos/repoinfo.xml | 12 --
ambari-server/src/test/python/TestOSCheck.py | 37 +++--
8 files changed, 192 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-common/src/main/python/ambari_commons/os_check.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_check.py b/ambari-common/src/main/python/ambari_commons/os_check.py
index c5457bb..b430c86 100644
--- a/ambari-common/src/main/python/ambari_commons/os_check.py
+++ b/ambari-common/src/main/python/ambari_commons/os_check.py
@@ -56,6 +56,8 @@ RESOURCES_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "resou
# family JSON data
OSFAMILY_JSON_RESOURCE = "os_family.json"
+JSON_OS_MAPPING = "mapping"
+JSON_OS_ALIASES = "aliases"
JSON_OS_TYPE = "distro"
JSON_OS_VERSION = "versions"
JSON_EXTENDS = "extends"
@@ -76,6 +78,8 @@ VER_NT_SERVER = 3
_IS_ORACLE_LINUX = os.path.exists('/etc/oracle-release')
_IS_REDHAT_LINUX = os.path.exists('/etc/redhat-release')
+SYSTEM_RELEASE_FILE = "/etc/system-release"
+
def _is_oracle_linux():
return _IS_ORACLE_LINUX
@@ -84,16 +88,16 @@ def _is_redhat_linux():
def advanced_check(distribution):
distribution = list(distribution)
- if os.path.exists("/etc/issue"):
- with open("/etc/issue", "rb") as fp:
+ if os.path.exists(SYSTEM_RELEASE_FILE):
+ with open(SYSTEM_RELEASE_FILE, "rb") as fp:
issue_content = fp.read()
if "Amazon" in issue_content:
distribution[0] = "amazon"
- search_groups = re.search('(\d+)\.(\d+)', issue_content)
+ search_groups = re.search('(\d+\.\d+)', issue_content)
if search_groups:
- distribution[1] = search_groups.group(1) # if version is 2015.09 only get 2015.
+ distribution[1] = search_groups.group(1)
return tuple(distribution)
@@ -114,16 +118,24 @@ class OS_CONST_TYPE(type):
f = open(os.path.join(RESOURCES_DIR, OSFAMILY_JSON_RESOURCE))
json_data = eval(f.read())
f.close()
- for family in json_data:
+
+ if JSON_OS_MAPPING not in json_data:
+ raise Exception("Invalid {0}".format(OSFAMILY_JSON_RESOURCE))
+
+ json_mapping_data = json_data[JSON_OS_MAPPING]
+
+ for family in json_mapping_data:
cls.FAMILY_COLLECTION += [family]
- cls.OS_COLLECTION += json_data[family][JSON_OS_TYPE]
+ cls.OS_COLLECTION += json_mapping_data[family][JSON_OS_TYPE]
cls.OS_FAMILY_COLLECTION += [{
'name': family,
- 'os_list': json_data[family][JSON_OS_TYPE]
+ 'os_list': json_mapping_data[family][JSON_OS_TYPE]
}]
- if JSON_EXTENDS in json_data[family]:
- cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_data[family][JSON_EXTENDS]
+ if JSON_EXTENDS in json_mapping_data[family]:
+ cls.OS_FAMILY_COLLECTION[-1][JSON_EXTENDS] = json_mapping_data[family][JSON_EXTENDS]
+
+ cls.OS_TYPE_ALIASES = json_data[JSON_OS_ALIASES] if JSON_OS_ALIASES in json_data else {}
except:
raise Exception("Couldn't load '%s' file" % OSFAMILY_JSON_RESOURCE)
@@ -194,7 +206,24 @@ class OSCheck:
distribution = ("Darwin", "TestOnly", "1.1.1", "1.1.1", "1.1")
return distribution
-
+
+ @staticmethod
+ def get_alias(os_type, os_version):
+ version_parts = os_version.split('.')
+ full_os_and_major_version = os_type + version_parts[0]
+
+ if full_os_and_major_version in OSConst.OS_TYPE_ALIASES:
+ alias = OSConst.OS_TYPE_ALIASES[full_os_and_major_version]
+ re_groups = re.search('(\D+)(\d+)$', alias).groups()
+ os_type = re_groups[0]
+ os_major_version = re_groups[1]
+
+ version_parts[0] = os_major_version
+ os_version = '.'.join(version_parts)
+
+ return os_type, os_version
+
+
@staticmethod
def get_os_type():
"""
@@ -205,6 +234,10 @@ class OSCheck:
In case cannot detect - exit.
"""
+ return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[0]
+
+ @staticmethod
+ def _get_os_type():
# Read content from /etc/*-release file
# Full release name
dist = OSCheck.os_distribution()
@@ -212,18 +245,18 @@ class OSCheck:
# special cases
if _is_oracle_linux():
- return 'oraclelinux'
+ operatingSystem = 'oraclelinux'
elif operatingSystem.startswith('suse linux enterprise server'):
- return 'sles'
+ operatingSystem = 'sles'
elif operatingSystem.startswith('red hat enterprise linux'):
- return 'redhat'
+ operatingSystem = 'redhat'
elif operatingSystem.startswith('darwin'):
- return 'mac'
+ operatingSystem = 'mac'
- if operatingSystem != '':
- return operatingSystem
- else:
+ if operatingSystem == '':
raise Exception("Cannot detect os type. Exiting...")
+
+ return operatingSystem
@staticmethod
def get_os_family():
@@ -257,11 +290,15 @@ class OSCheck:
In case cannot detect raises exception.
"""
+ return OSCheck.get_alias(OSCheck._get_os_type(), OSCheck._get_os_version())[1]
+
+ @staticmethod
+ def _get_os_version():
# Read content from /etc/*-release file
# Full release name
dist = OSCheck.os_distribution()
dist = dist[1]
-
+
if dist:
return dist
else:
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-common/src/main/python/ambari_commons/resources/os_family.json
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/resources/os_family.json b/ambari-common/src/main/python/ambari_commons/resources/os_family.json
index 13014fc..1558c1b 100644
--- a/ambari-common/src/main/python/ambari_commons/resources/os_family.json
+++ b/ambari-common/src/main/python/ambari_commons/resources/os_family.json
@@ -1,72 +1,69 @@
{
- "redhat": {
- "distro": [
- "redhat",
- "fedora",
- "centos",
- "oraclelinux",
- "ascendos",
- "xenserver",
- "oel",
- "ovs",
- "cloudlinux",
- "slc",
- "scientific",
- "psbm",
- "centos linux"
- ],
- "versions": [
- 6,
- 7
- ]
- },
- "amazon": {
- "extends" : "redhat",
- "distro": [
- "amazon"
- ],
- "versions": [
- 2015
- ]
- },
- "debian": {
- "extends" : "ubuntu",
- "distro": [
- "debian"
- ],
- "versions": [
- 7
- ]
- },
- "ubuntu": {
- "distro": [
- "ubuntu"
- ],
- "versions": [
- 12,
- 14
- ]
- },
- "suse": {
- "distro": [
- "sles",
- "sled",
- "opensuse",
- "suse"
- ],
- "versions": [
- 11
- ]
- },
- "winsrv": {
- "distro": [
- "win2008server",
- "win2008serverr2",
- "win2012server",
- "win2012serverr2"
- ],
- "versions": [
- 6
- ]
- }
+ "mapping": {
+ "redhat": {
+ "distro": [
+ "redhat",
+ "fedora",
+ "centos",
+ "oraclelinux",
+ "amazon",
+ "ascendos",
+ "xenserver",
+ "oel",
+ "ovs",
+ "cloudlinux",
+ "slc",
+ "scientific",
+ "psbm",
+ "centos linux"
+ ],
+ "versions": [
+ 6,
+ 7
+ ]
+ },
+ "debian": {
+ "extends" : "ubuntu",
+ "distro": [
+ "debian"
+ ],
+ "versions": [
+ 7
+ ]
+ },
+ "ubuntu": {
+ "distro": [
+ "ubuntu"
+ ],
+ "versions": [
+ 12,
+ 14
+ ]
+ },
+ "suse": {
+ "distro": [
+ "sles",
+ "sled",
+ "opensuse",
+ "suse"
+ ],
+ "versions": [
+ 11
+ ]
+ },
+ "winsrv": {
+ "distro": [
+ "win2008server",
+ "win2008serverr2",
+ "win2012server",
+ "win2012serverr2"
+ ],
+ "versions": [
+ 6
+ ]
+ }
+ },
+ "aliases": {
+ "amazon2015": "amazon6"
+ }
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java
new file mode 100644
index 0000000..3f9158f
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/JsonOsFamilyRoot.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack;
+
+import java.util.Map;
+
+public class JsonOsFamilyRoot {
+ private Map<String, JsonOsFamilyEntry> mapping;
+ private Map<String, String> aliases;
+
+ public Map<String, JsonOsFamilyEntry> getMapping() {
+ return mapping;
+ }
+ public void setMapping(Map<String, JsonOsFamilyEntry> mapping) {
+ this.mapping = mapping;
+ }
+ public Map<String, String> getAliases() {
+ return aliases;
+ }
+ public void setAliases(Map<String, String> aliases) {
+ this.aliases = aliases;
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
index 37a6db3..e494c44 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/OsFamily.java
@@ -48,11 +48,14 @@ public class OsFamily {
private final String os_pattern = "([\\D]+|(?:[\\D]+[\\d]+[\\D]+))([\\d]*)";
private final String OS_DISTRO = "distro";
private final String OS_VERSION = "versions";
+ private final String OS_MAPPING = "mapping";
+ private final String OS_ALIASES = "aliases";
private final String LOAD_CONFIG_MSG = "Could not load OS family definition from %s file";
private final String FILE_NAME = "os_family.json";
private final Logger LOG = LoggerFactory.getLogger(OsFamily.class);
private Map<String, JsonOsFamilyEntry> osMap = null;
+ private JsonOsFamilyRoot jsonOsFamily = null;
/**
* Initialize object
@@ -77,9 +80,10 @@ public class OsFamily {
if (!f.exists()) throw new Exception();
inputStream = new FileInputStream(f);
- Type type = new TypeToken<Map<String, JsonOsFamilyEntry>>() {}.getType();
+ Type type = new TypeToken<JsonOsFamilyRoot>() {}.getType();
Gson gson = new Gson();
- osMap = gson.fromJson(new InputStreamReader(inputStream), type);
+ jsonOsFamily = gson.fromJson(new InputStreamReader(inputStream), type);
+ osMap = jsonOsFamily.getMapping();
} catch (Exception e) {
LOG.error(String.format(LOAD_CONFIG_MSG, new File(SharedResourcesPath, FILE_NAME).toString()));
throw new RuntimeException(e);
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml
index 9decf51..dbf8506 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/repos/repoinfo.xml
@@ -29,18 +29,6 @@
<reponame>HDP-UTILS</reponame>
</repo>
</os>
- <os family="amazon2015">
- <repo>
- <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.2.6.0</baseurl>
- <repoid>HDP-2.2</repoid>
- <reponame>HDP</reponame>
- </repo>
- <repo>
- <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
- <repoid>HDP-UTILS-1.1.0.20</repoid>
- <reponame>HDP-UTILS</reponame>
- </repo>
- </os>
<os family="suse11">
<repo>
<baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.2.6.0</baseurl>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml
index 279134b..142b87d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/repos/repoinfo.xml
@@ -41,18 +41,6 @@
<reponame>HDP-UTILS</reponame>
</repo>
</os>
- <os family="amazon2015">
- <repo>
- <baseurl>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.3.0.0</baseurl>
- <repoid>HDP-2.3</repoid>
- <reponame>HDP</reponame>
- </repo>
- <repo>
- <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
- <repoid>HDP-UTILS-1.1.0.20</repoid>
- <reponame>HDP-UTILS</reponame>
- </repo>
- </os>
<os family="suse11">
<repo>
<baseurl>http://public-repo-1.hortonworks.com/HDP/suse11sp3/2.x/updates/2.3.0.0</baseurl>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml
index 6ac43f9..54bd3da 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.4/repos/repoinfo.xml
@@ -41,18 +41,6 @@
<reponame>HDP-UTILS</reponame>
</repo>
</os>
- <os family="redhat2015">
- <repo>
- <baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/centos6/2.x/updates/2.4.0.0</baseurl>
- <repoid>HDP-2.4</repoid>
- <reponame>HDP</reponame>
- </repo>
- <repo>
- <baseurl>http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.20/repos/centos6</baseurl>
- <repoid>HDP-UTILS-1.1.0.20</repoid>
- <reponame>HDP-UTILS</reponame>
- </repo>
- </os>
<os family="suse11">
<repo>
<baseurl>http://s3.amazonaws.com/dev.hortonworks.com/HDP/suse11sp3/2.x/updates/2.4.0.0</baseurl>
http://git-wip-us.apache.org/repos/asf/ambari/blob/6270fe39/ambari-server/src/test/python/TestOSCheck.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/TestOSCheck.py b/ambari-server/src/test/python/TestOSCheck.py
index cf114a1..d919fbc 100644
--- a/ambari-server/src/test/python/TestOSCheck.py
+++ b/ambari-server/src/test/python/TestOSCheck.py
@@ -30,7 +30,7 @@ from mock.mock import MagicMock
from only_for_platform import os_distro_value, os_distro_value_linux
-from ambari_commons import OSCheck
+from ambari_commons import OSCheck, OSConst
import os_check_type
utils = __import__('ambari_server.utils').utils
@@ -50,7 +50,7 @@ class TestOSCheck(TestCase):
# 1 - Any system
mock_is_oracle_linux.return_value = False
- mock_linux_distribution.return_value = ('my_os', '', '')
+ mock_linux_distribution.return_value = ('my_os', '2015.09', '')
result = OSCheck.get_os_type()
self.assertEquals(result, 'my_os')
@@ -66,13 +66,13 @@ class TestOSCheck(TestCase):
# 3 - path exist: '/etc/oracle-release'
mock_is_oracle_linux.return_value = True
- mock_linux_distribution.return_value = ('some_os', '', '')
+ mock_linux_distribution.return_value = ('some_os', '1234', '')
result = OSCheck.get_os_type()
self.assertEquals(result, 'oraclelinux')
# 4 - Common system
mock_is_oracle_linux.return_value = False
- mock_linux_distribution.return_value = ('CenToS', '', '')
+ mock_linux_distribution.return_value = ('CenToS', '4.56', '')
result = OSCheck.get_os_type()
self.assertEquals(result, 'centos')
@@ -99,31 +99,31 @@ class TestOSCheck(TestCase):
# 1 - Any system
mock_exists.return_value = False
- mock_linux_distribution.return_value = ('MY_os', '', '')
+ mock_linux_distribution.return_value = ('MY_os', '5.6.7', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'my_os')
# 2 - Redhat
mock_exists.return_value = False
- mock_linux_distribution.return_value = ('Centos Linux', '', '')
+ mock_linux_distribution.return_value = ('Centos Linux', '2.4', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'redhat')
# 3 - Ubuntu
mock_exists.return_value = False
- mock_linux_distribution.return_value = ('Ubuntu', '', '')
+ mock_linux_distribution.return_value = ('Ubuntu', '14.04', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'ubuntu')
# 4 - Suse
mock_exists.return_value = False
mock_linux_distribution.return_value = (
- 'suse linux enterprise server', '', '')
+ 'suse linux enterprise server', '11.3', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'suse')
mock_exists.return_value = False
- mock_linux_distribution.return_value = ('SLED', '', '')
+ mock_linux_distribution.return_value = ('SLED', '1.2.3.4.5', '')
result = OSCheck.get_os_family()
self.assertEquals(result, 'suse')
@@ -141,7 +141,7 @@ class TestOSCheck(TestCase):
def test_get_os_version(self, mock_linux_distribution):
# 1 - Any system
- mock_linux_distribution.return_value = ('', '123.45', '')
+ mock_linux_distribution.return_value = ('some_os', '123.45', '')
result = OSCheck.get_os_version()
self.assertEquals(result, '123.45')
@@ -159,7 +159,7 @@ class TestOSCheck(TestCase):
def test_get_os_major_version(self, mock_linux_distribution):
# 1
- mock_linux_distribution.return_value = ('', '123.45.67', '')
+ mock_linux_distribution.return_value = ('abcd_os', '123.45.67', '')
result = OSCheck.get_os_major_version()
self.assertEquals(result, '123')
@@ -167,6 +167,21 @@ class TestOSCheck(TestCase):
mock_linux_distribution.return_value = ('Suse', '11', '')
result = OSCheck.get_os_major_version()
self.assertEquals(result, '11')
+
+ @patch.object(OSCheck, "os_distribution")
+ def test_aliases(self, mock_linux_distribution):
+ OSConst.OS_TYPE_ALIASES['qwerty_os123'] = 'aliased_os5'
+ OSConst.OS_FAMILY_COLLECTION.append({
+ 'name': 'aliased_os_family',
+ 'os_list': ["aliased_os"]
+ })
+
+ mock_linux_distribution.return_value = ('qwerty_os', '123.45.67', '')
+
+ self.assertEquals(OSCheck.get_os_type(), 'aliased_os')
+ self.assertEquals(OSCheck.get_os_major_version(), '5')
+ self.assertEquals(OSCheck.get_os_version(), '5.45.67')
+ self.assertEquals(OSCheck.get_os_family(), 'aliased_os_family')
@patch.object(OSCheck, "os_distribution")
def test_get_os_release_name(self, mock_linux_distribution):