You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/06/18 01:13:14 UTC
git commit: SLIDER-139. Ensure python unit tests use the correct
version of python
Repository: incubator-slider
Updated Branches:
refs/heads/develop 055ed0b07 -> 3b2f5f6bb
SLIDER-139. Ensure python unit tests use the correct version of python
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3b2f5f6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3b2f5f6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3b2f5f6b
Branch: refs/heads/develop
Commit: 3b2f5f6bbd56c485b6dbbd37f19ee1872dc6fbb7
Parents: 055ed0b
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jun 17 16:11:06 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Jun 17 16:11:06 2014 -0700
----------------------------------------------------------------------
slider-agent/pom.xml | 2 +-
.../libraries/functions/os_check.py | 172 ++++++++++++++-----
.../libraries/script/hook.py | 2 +-
.../src/test/python/agent/TestPythonExecutor.py | 2 +-
slider-agent/src/test/python/python-wrap | 40 +++++
.../TestPropertiesFileResource.py | 2 +-
.../apache/slider/common/tools/SliderUtils.java | 29 +++-
7 files changed, 191 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-agent/pom.xml
----------------------------------------------------------------------
diff --git a/slider-agent/pom.xml b/slider-agent/pom.xml
index 06f99b5..efc61ef 100644
--- a/slider-agent/pom.xml
+++ b/slider-agent/pom.xml
@@ -67,7 +67,7 @@
<executions>
<execution>
<configuration>
- <executable>python</executable>
+ <executable>${project.basedir}/src/test/python/python-wrap</executable>
<workingDirectory>src/test/python</workingDirectory>
<arguments>
<argument>unitTests.py</argument>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
index 7a72bc8..abfceb8 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/os_check.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.6
+#!/usr/bin/env python
'''
Licensed to the Apache Software Foundation (ASF) under one
@@ -26,11 +26,70 @@ __all__ = [
'OSCheck',
]
-class OSCheck(object):
- def __init__(self):
- pass
- def get_os_type(self):
+def linux_distribution():
+ PYTHON_VER = sys.version_info[0] * 10 + sys.version_info[1]
+
+ if PYTHON_VER < 26:
+ linux_distribution = platform.dist()
+ else:
+ linux_distribution = platform.linux_distribution()
+
+ return linux_distribution
+
+
+class OS_CONST_TYPE(type):
+ # os families
+ REDHAT_FAMILY = 'redhat'
+ DEBIAN_FAMILY = 'debian'
+ SUSE_FAMILY = 'suse'
+
+ # Declare here os type mapping
+ OS_FAMILY_COLLECTION = [
+ {'name': REDHAT_FAMILY,
+ 'os_list':
+ ['redhat', 'fedora', 'centos', 'oraclelinux',
+ 'ascendos', 'amazon', 'xenserver', 'oel', 'ovs',
+ 'cloudlinux', 'slc', 'scientific', 'psbm',
+ 'centos linux']
+ },
+ {'name': DEBIAN_FAMILY,
+ 'os_list': ['ubuntu', 'debian']
+ },
+ {'name': SUSE_FAMILY,
+ 'os_list': ['sles', 'sled', 'opensuse', 'suse']
+ }
+ ]
+ # Would be generated from Family collection definition
+ OS_COLLECTION = []
+
+ def __init__(cls, name, bases, dct):
+ for item in cls.OS_FAMILY_COLLECTION:
+ cls.OS_COLLECTION += item['os_list']
+
+ def __getattr__(cls, name):
+ """
+ Added support of class.OS_<os_type> properties defined in OS_COLLECTION
+ Example:
+ OSConst.OS_CENTOS would return centos
+ OSConst.OS_OTHEROS would triger an error, coz
+ that os is not present in OS_FAMILY_COLLECTION map
+ """
+ name = name.lower()
+ if "os_" in name and name[3:] in cls.OS_COLLECTION:
+ return name[3:]
+ else:
+ raise Exception("Unknown class property '%s'" % name)
+
+
+class OSConst:
+ __metaclass__ = OS_CONST_TYPE
+
+
+class OSCheck:
+
+ @staticmethod
+ def get_os_type():
"""
Return values:
redhat, fedora, centos, oraclelinux, ascendos,
@@ -41,7 +100,7 @@ class OSCheck(object):
"""
# Read content from /etc/*-release file
# Full release name
- dist = platform.linux_distribution()
+ dist = linux_distribution()
operatingSystem = dist[0].lower()
# special cases
@@ -55,31 +114,26 @@ class OSCheck(object):
if operatingSystem != '':
return operatingSystem
else:
- print "Cannot detect os type. Exiting..."
- sys.exit(1)
-
+ raise Exception("Cannot detect os type. Exiting...")
- def get_os_family(self):
+ @staticmethod
+ def get_os_family():
"""
Return values:
redhat, debian, suse ... and others
In case cannot detect raises exception( from self.get_operating_system_type() ).
"""
- os_family = self.get_os_type()
- if os_family in ['redhat', 'fedora', 'centos', 'oraclelinux', 'ascendos',
- 'amazon', 'xenserver', 'oel', 'ovs', 'cloudlinux',
- 'slc', 'scientific', 'psbm', 'centos linux']:
- os_family = 'RedHat'
- elif os_family in ['ubuntu', 'debian']:
- os_family = 'Debian'
- elif os_family in ['sles', 'sled', 'opensuse', 'suse']:
- os_family = 'Suse'
- #else: os_family = self.get_os_type()
- return os_family.lower()
+ os_family = OSCheck.get_os_type()
+ for os_family_item in OSConst.OS_FAMILY_COLLECTION:
+ if os_family in os_family_item['os_list']:
+ os_family = os_family_item['name']
+ break
+ return os_family.lower()
- def get_os_version(self):
+ @staticmethod
+ def get_os_version():
"""
Returns the OS version
@@ -87,57 +141,81 @@ class OSCheck(object):
"""
# Read content from /etc/*-release file
# Full release name
- dist = platform.linux_distribution()
+ dist = linux_distribution()
dist = dist[1]
if dist:
return dist
else:
- print "Cannot detect os version. Exiting..."
- sys.exit(1)
+ raise Exception("Cannot detect os version. Exiting...")
- def get_os_major_version(self):
+ @staticmethod
+ def get_os_major_version():
"""
Returns the main OS version like
Centos 6.5 --> 6
RedHat 1.2.3 --> 1
"""
- return self.get_os_version().split('.')[0]
+ return OSCheck.get_os_version().split('.')[0]
- def get_os_release_name(self):
+ @staticmethod
+ def get_os_release_name():
"""
Returns the OS release name
In case cannot detect raises exception.
"""
- dist = platform.linux_distribution()
+ dist = linux_distribution()
dist = dist[2].lower()
if dist:
return dist
else:
- print "Cannot detect os release name. Exiting..."
- sys.exit(1)
+ raise Exception("Cannot detect os release name. Exiting...")
+ # Exception safe family check functions
-def main(argv=None):
- # Same logic that was in "os_type_check.sh"
- if len(sys.argv) != 2:
- print "Usage: <cluster_os>"
- sys.exit(2)
- pass
+ @staticmethod
+ def is_debian_family():
+ """
+ Return true if it is so or false if not
+
+ This is safe check for debian family, doesn't generate exception
+ """
+ try:
+ if OSCheck.get_os_family() == OSConst.DEBIAN_FAMILY:
+ return True
+ except Exception:
+ pass
+ return False
+
+ @staticmethod
+ def is_suse_family():
+ """
+ Return true if it is so or false if not
- cluster_os = sys.argv[1]
- current_os = OSCheck().get_os_family() + OSCheck().get_os_major_version()
+ This is safe check for suse family, doesn't generate exception
+ """
+ try:
+ if OSCheck.get_os_family() == OSConst.SUSE_FAMILY:
+ return True
+ except Exception:
+ pass
+ return False
+
+ @staticmethod
+ def is_redhat_family():
+ """
+ Return true if it is so or false if not
- # If agent/server have the same {"family","main_version"} - then ok.
- print "Cluster primary/cluster OS type is %s and local/current OS type is %s" % (
- cluster_os, current_os)
- if current_os == cluster_os:
- sys.exit(0)
- else:
- print "Local OS is not compatible with cluster primary OS. Please perform manual bootstrap on this host."
- sys.exit(1)
+ This is safe check for redhat family, doesn't generate exception
+ """
+ try:
+ if OSCheck.get_os_family() == OSConst.REDHAT_FAMILY:
+ return True
+ except Exception:
+ pass
+ return False
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/hook.py b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
index 399baac..acf4872 100644
--- a/slider-agent/src/main/python/resource_management/libraries/script/hook.py
+++ b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python2.6
+#!/usr/bin/env python
'''
Licensed to the Apache Software Foundation (ASF) under one
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-agent/src/test/python/agent/TestPythonExecutor.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestPythonExecutor.py b/slider-agent/src/test/python/agent/TestPythonExecutor.py
index 7d0eaaf..1b12a0a 100644
--- a/slider-agent/src/test/python/agent/TestPythonExecutor.py
+++ b/slider-agent/src/test/python/agent/TestPythonExecutor.py
@@ -145,7 +145,7 @@ class TestPythonExecutor(TestCase):
executor = PythonExecutor("/tmp", AgentConfig("", ""))
command = executor.python_command("script", ["script_param1"])
self.assertEqual(4, len(command))
- self.assertTrue("python" in command[0])
+ self.assertTrue("python" in command[0].lower(), "Looking for python in %s" % (command[0].lower()))
self.assertEquals("-S", command[1])
self.assertEquals("script", command[2])
self.assertEquals("script_param1", command[3])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-agent/src/test/python/python-wrap
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap
new file mode 100755
index 0000000..40dc785
--- /dev/null
+++ b/slider-agent/src/test/python/python-wrap
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Copyright 2011 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export PYTHONPATH=/usr/lib/python2.6/site-packages/common_functions:$PYTHONPATH
+
+# reset settings
+unset PYTHON
+
+# checking for preferable python versions
+if [ -a /usr/bin/python2.7 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.7
+fi
+
+if [ -a /usr/bin/python2.6 ] && [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python2.6
+fi
+
+# if no preferable python versions found, try to use system one
+if [[ -z "$PYTHON" ]]; then
+ PYTHON=/usr/bin/python
+fi
+
+# execute script
+$PYTHON "$@"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
index 79aef58..6eb01cf 100644
--- a/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
+++ b/slider-agent/src/test/python/resource_management/TestPropertiesFileResource.py
@@ -29,7 +29,7 @@ from resource_management.core.system import System
from resource_management.libraries import PropertiesFile
@patch.object(System, "os_family", new='redhat')
-class TestPropertiesFIleResource(TestCase):
+class TestPropertiesFileResource(TestCase):
"""
PropertiesFile="resource_management.libraries.providers.properties_file.PropertiesFileProvider"
Testing PropertiesFile(PropertiesFileProvider) with different 'properties dictionary'
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b2f5f6b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 8dd3eb6..b97ff63 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -20,6 +20,7 @@ package org.apache.slider.common.tools;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
+import org.apache.commons.io.output.ByteArrayOutputStream;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -1430,8 +1431,8 @@ public final class SliderUtils {
}
public static InputStream getApplicationResourceInputStream(FileSystem fs,
- Path appPath,
- String entry)
+ Path appPath,
+ String entry)
throws IOException {
InputStream is = null;
FSDataInputStream appStream = fs.open(appPath);
@@ -1441,12 +1442,26 @@ public final class SliderUtils {
while (!done && (zipEntry = zis.getNextZipEntry()) != null) {
if (entry.equals(zipEntry.getName())) {
int size = (int) zipEntry.getSize();
- byte[] content = new byte[size];
- int offset = 0;
- while (offset < size) {
- offset += zis.read(content, offset, size - offset);
+ if (size != -1) {
+ log.info("Reading {} of size {}", zipEntry.getName(), zipEntry.getSize());
+ byte[] content = new byte[size];
+ int offset = 0;
+ while (offset < size) {
+ offset += zis.read(content, offset, size - offset);
+ }
+ is = new ByteArrayInputStream(content);
+ } else {
+ log.info("Size unknown. Reading {}", zipEntry.getName());
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ while (true) {
+ int byteRead = zis.read();
+ if (byteRead == -1) {
+ break;
+ }
+ baos.write(byteRead);
+ }
+ is = new ByteArrayInputStream(baos.toByteArray());
}
- is = new ByteArrayInputStream(content);
done = true;
}
}