You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2016/02/05 14:44:11 UTC
ambari git commit: AMBARI-14939. [Ambari tarballs] optionally install
required packages (aonishuk)
Repository: ambari
Updated Branches:
refs/heads/trunk 691958665 -> ed492292c
AMBARI-14939. [Ambari tarballs] optionally install required packages (aonishuk)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/ed492292
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/ed492292
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/ed492292
Branch: refs/heads/trunk
Commit: ed492292cd4fedd41ebe95ba97a73e57cceb0bae
Parents: 6919586
Author: Andrew Onishuk <ao...@hortonworks.com>
Authored: Fri Feb 5 15:44:04 2016 +0200
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Fri Feb 5 15:44:04 2016 +0200
----------------------------------------------------------------------
ambari-agent/pom.xml | 38 ++++--
.../src/main/package/dependencies.properties | 32 +++++
.../src/main/repo/install_ambari_tarball.py | 117 ++++++++++++++++++-
ambari-server/pom.xml | 40 +++++--
.../src/main/package/dependencies.properties | 33 ++++++
5 files changed, 239 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-agent/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/pom.xml b/ambari-agent/pom.xml
index d049a6c..c2c993f 100644
--- a/ambari-agent/pom.xml
+++ b/ambari-agent/pom.xml
@@ -43,10 +43,7 @@
<jinja.install.dir>/usr/lib/ambari-agent/lib/ambari_jinja2</jinja.install.dir>
<simplejson.install.dir>/usr/lib/ambari-agent/lib/ambari_simplejson</simplejson.install.dir>
<lib.dir>/usr/lib/ambari-agent/lib</lib.dir>
- <python.ver>python >= 2.6</python.ver>
- <deb.python.ver>python (>= 2.6)</deb.python.ver>
<deb.architecture>amd64</deb.architecture>
- <deb.dependency.list>openssl, zlibc, ${deb.python.ver}</deb.dependency.list>
<ambari.server.module>../ambari-server</ambari.server.module>
<target.cache.dir>${project.build.directory}/cache/</target.cache.dir>
<resource.keeper.script>${ambari.server.module}/src/main/python/ambari_server/resourceFilesKeeper.py</resource.keeper.script>
@@ -140,6 +137,23 @@
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <files>
+ <file>${basedir}/src/main/package/dependencies.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>2.1.4</version>
<executions>
@@ -156,10 +170,7 @@
<group>Development</group>
<description>Maven Recipe: RPM Package.</description>
<requires>
- <require>openssl</require>
- <require>rpm-python</require>
- <require>zlib</require>
- <require>${python.ver}</require>
+ <require>${rpm.dependency.list}</require>
</requires>
<postinstallScriptlet>
<scriptFile>src/main/package/rpm/postinstall.sh</scriptFile>
@@ -355,6 +366,19 @@
<resource>
<directory>${basedir}/src/main/package/deb/control</directory>
</resource>
+ <resource>
+ <directory>${basedir}/src/main/package</directory>
+ <includes>
+ <include>dependencies.properties</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory>
+ <includes>
+ <include>os_check.py</include>
+ <include>resources/os_family.json</include>
+ </includes>
+ </resource>
</resources>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-agent/src/main/package/dependencies.properties
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/package/dependencies.properties b/ambari-agent/src/main/package/dependencies.properties
new file mode 100644
index 0000000..07b0b68
--- /dev/null
+++ b/ambari-agent/src/main/package/dependencies.properties
@@ -0,0 +1,32 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Please make sure the format of the file is one of the following
+# rpm.dependency.list=...
+# rpm.dependency.list.[os_family_name]=...
+# rpm.dependency.list.[os_family_name][starting_from_version]=...
+# deb.dependency.list=...
+# deb.dependency.list.[os_family_name]=...
+# deb.dependency.list.[os_family_name][starting_from_version]=...
+#
+# Examples:
+# rpm.dependency.list.suse=openssl,\nRequires: rpm-python # for all suse family os
+# rpm.dependency.list.suse11=curl # for all suse family os higher or equal to 11.
+#
+# Such a format is respected by install_ambari_tarball.py by default,
+# however should be encouraged manually in pom.xml.
+
+rpm.dependency.list=openssl,\nRequires: rpm-python,\nRequires: zlib,\nRequires: python >= 2.6
+deb.dependency.list=openssl, zlibc, python (>= 2.6)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-common/src/main/repo/install_ambari_tarball.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/repo/install_ambari_tarball.py b/ambari-common/src/main/repo/install_ambari_tarball.py
index aa33ede..5c65b60 100644
--- a/ambari-common/src/main/repo/install_ambari_tarball.py
+++ b/ambari-common/src/main/repo/install_ambari_tarball.py
@@ -17,11 +17,13 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+import re
import os
import sys
import logging
import subprocess
from optparse import OptionParser
+import ConfigParser
USAGE = "Usage: %prog [OPTION]... URL"
DESCRIPTION = "URL should point to full tar.gz location e.g.: https://public-repo-1.hortonworks.com/something/ambari-server.tar.gz"
@@ -32,9 +34,16 @@ PREINST_SCRIPT = "preinst"
PRERM_SCRIPT = "prerm"
POSTINST_SCRIPT = "postinst"
POSTRM_SCRIPT = "postrm"
+OS_CHECK = "os_check.py"
+OS_PACKAGE_DEPENDENCIES = "dependencies.properties"
+OS_FAMILY_DESCRIPTION = "resources/os_family.json"
+RPM_DEPENDENCIES_PROPERTY = "rpm.dependency.list"
+DEB_DEPENDENCIES_PROPERTY = "deb.dependency.list"
-ROOT_FOLDER_ENV_VARIABLE = "AMBARI_ROOT_FOLDER"
+FILES_TO_DOWNLOAD = [PREINST_SCRIPT, PRERM_SCRIPT, POSTINST_SCRIPT, POSTRM_SCRIPT, OS_CHECK, OS_FAMILY_DESCRIPTION, OS_PACKAGE_DEPENDENCIES]
+ROOT_FOLDER_ENV_VARIABLE = "AMBARI_ROOT_FOLDER"
+
class Utils:
verbose = False
@staticmethod
@@ -65,26 +74,72 @@ class Utils:
return out
+ @staticmethod
+ def install_package(name):
+ from os_check import OSCheck
+
+ logger.info("Checking for existance of {0} dependency package".format(name))
+ is_rpm = not OSCheck.is_ubuntu_family()
+
+ if is_rpm:
+ is_installed_cmd = ['rpm', '-q'] + [name]
+ install_cmd = ['sudo', 'yum', '-y', 'install'] + [name]
+ else:
+ is_installed_cmd = ['dpkg', '-s'] + [name]
+ install_cmd = ['sudo', 'apt-get', '-y', 'install'] + [name]
+
+ try:
+ Utils.os_call(is_installed_cmd, logoutput=False)
+ logger.info("Package {0} is already installed. Skipping installation.".format(name))
+ except OsCallFailure:
+ logger.info("Package {0} is not installed. Installing it...".format(name))
+ Utils.os_call(install_cmd)
+
+
+class FakePropertiesHeader(object):
+ """
+ Hacky class to parse properties file without sections.
+ see http://stackoverflow.com/questions/2819696/module-to-use-when-parsing-properties-file-in-python/2819788#2819788
+ """
+ FAKE_SECTION_NAME = 'section'
+ def __init__(self, fp):
+ self.fp = fp
+ self.sechead = '[{0}]\n'.format(FakePropertiesHeader.FAKE_SECTION_NAME)
+ def readline(self):
+ if self.sechead:
+ try:
+ return self.sechead
+ finally:
+ self.sechead = None
+ else:
+ return self.fp.readline()
+
class OsCallFailure(RuntimeError):
pass
class Installer:
- def __init__(self, archive_url, root_folder, verbose):
+ def __init__(self, archive_url, root_folder, verbose, skip_dependencies):
splited_url = archive_url.split('/')
self.archive_name = splited_url[-1]
self.base_url = '/'.join(splited_url[0:-1])
self.root_folder = root_folder
self.verbose = verbose
+ self.skip_dependencies = skip_dependencies
- def download_files(self):
- for name in [ self.archive_name, PREINST_SCRIPT, PRERM_SCRIPT, POSTINST_SCRIPT, POSTRM_SCRIPT]:
+ def download_files(self, files_list):
+ for name in files_list:
+ dirname = os.path.dirname(name)
+ if dirname:
+ Utils.os_call(["mkdir", "-p", dirname])
+
url = "{0}/{1}".format(self.base_url, name)
logger.info("Downloading {0}".format(url))
Utils.os_call(["wget", "-O", name, url])
def run(self):
- self.download_files()
+ self.download_files([self.archive_name] +FILES_TO_DOWNLOAD) # [self.archive_name] +
+ self.check_dependencies()
self.run_script(PRERM_SCRIPT, ["remove"]) # in case we are upgrading
self.run_script(POSTRM_SCRIPT, ["remove"]) # in case we are upgrading
@@ -92,6 +147,54 @@ class Installer:
self.extract_archive()
self.run_script(POSTINST_SCRIPT, ["configure"])
+ def check_dependencies(self):
+ from os_check import OSCheck
+
+ os_family = OSCheck.get_os_family()
+ os_version = OSCheck.get_os_major_version()
+
+ is_rpm = not OSCheck.is_ubuntu_family()
+ property_prefix = RPM_DEPENDENCIES_PROPERTY if is_rpm else DEB_DEPENDENCIES_PROPERTY
+
+ cp = ConfigParser.SafeConfigParser()
+ with open(OS_PACKAGE_DEPENDENCIES) as fp:
+ cp.readfp(FakePropertiesHeader(fp))
+
+ properties = dict(cp.items(FakePropertiesHeader.FAKE_SECTION_NAME))
+
+ packages_string = None
+ postfixes = [os_family+str(ver) for ver in range(int(os_version),0,-1)+['']]+['']
+ for postfix in postfixes:
+ property_name = property_prefix + postfix
+ if property_name in properties:
+ packages_string = properties[property_name]
+ break
+
+ if packages_string is None:
+ err_msg = "No os dependencies found. "
+ if self.skip_dependencies:
+ logger.warn(err_msg)
+ else:
+ raise Exception(err_msg)
+
+ packages_string = re.sub('Requires\s*:','',packages_string)
+ packages_string = re.sub('\\\\n','',packages_string)
+ packages_string = re.sub('\s','',packages_string)
+ packages_string = re.sub('[()]','',packages_string)
+
+ if self.skip_dependencies:
+ var = raw_input("Please confirm you have the following packages installed {0} (y/n): ".format(packages_string))
+ if var.lower() != "y" and var.lower() != "yes":
+ raise Exception("User canceled the installation.")
+ return
+
+ pacakges = packages_string.split(',')
+
+ for package in pacakges:
+ split_parts = re.split('[><=]', package)
+ package_name = split_parts[0]
+ Utils.install_package(package_name)
+
def run_script(self, script_name, args):
bash_args = []
if self.verbose:
@@ -112,6 +215,8 @@ class Runner:
help="sets output level to more detailed")
parser.add_option("-r", "--root-folder", dest="root_folder", default="/",
help="root folder to install Ambari to. E.g.: /opt")
+ parser.add_option("-d", "--dependencies-skip", dest="skip_dependencies", action="store_true",
+ help="the script won't install the package dependencies. Please make sure to install them manually.")
(self.options, args) = parser.parse_args()
@@ -138,7 +243,7 @@ class Runner:
Utils.verbose = self.options.verbose
# TODO: check if ends with tar.gz?
- targz_installer = TargzInstaller(self.url, self.options.root_folder, self.options.verbose)
+ targz_installer = TargzInstaller(self.url, self.options.root_folder, self.options.verbose, self.options.skip_dependencies)
targz_installer.run()
if __name__ == '__main__':
http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 623ddca..b500fb2 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -26,10 +26,7 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<python.ver>python >= 2.6</python.ver>
<!-- On centos the python xml's are inside python package -->
- <python.xml.package>${python.ver}</python.xml.package>
- <deb.python.ver>python (>= 2.6)</deb.python.ver>
<deb.architecture>amd64</deb.architecture>
- <deb.dependency.list>openssl, postgresql (>= 8.1), ${deb.python.ver}, curl</deb.dependency.list>
<custom.tests>false</custom.tests>
<hdpUrlForCentos6>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.1.0</hdpUrlForCentos6>
<hdpLatestUrl>http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json</hdpLatestUrl>
@@ -195,6 +192,19 @@
<resource>
<directory>${basedir}/src/main/package/deb/control</directory>
</resource>
+ <resource>
+ <directory>${basedir}/src/main/package</directory>
+ <includes>
+ <include>dependencies.properties</include>
+ </includes>
+ </resource>
+ <resource>
+ <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory>
+ <includes>
+ <include>os_check.py</include>
+ <include>resources/os_family.json</include>
+ </includes>
+ </resource>
</resources>
</configuration>
</execution>
@@ -309,6 +319,23 @@
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <files>
+ <file>${basedir}/src/main/package/dependencies.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
<artifactId>rpm-maven-plugin</artifactId>
<version>2.1.4</version>
<executions>
@@ -326,10 +353,7 @@
<description>Maven Recipe: RPM Package.</description>
<autoRequires>no</autoRequires>
<requires>
- <require>postgresql-server >= 8.1</require>
- <require>openssl</require>
- <require>${python.ver}</require>
- <require>${python.xml.package}</require>
+ <require>${rpm.dependency.list}</require>
</requires>
<postinstallScriptlet>
<scriptFile>src/main/package/rpm/postinstall.sh</scriptFile>
@@ -738,7 +762,7 @@
<profile>
<id>suse11</id>
<properties>
- <python.xml.package>python-xml</python.xml.package>
+ <rpm.dependency.list>${rpm.dependency.list.suse}</rpm.dependency.list>
</properties>
</profile>
<profile>
http://git-wip-us.apache.org/repos/asf/ambari/blob/ed492292/ambari-server/src/main/package/dependencies.properties
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/package/dependencies.properties b/ambari-server/src/main/package/dependencies.properties
new file mode 100644
index 0000000..01244a8
--- /dev/null
+++ b/ambari-server/src/main/package/dependencies.properties
@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information rega4rding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Please make sure the format of the file is one of the following
+# rpm.dependency.list=...
+# rpm.dependency.list.[os_family_name]=...
+# rpm.dependency.list.[os_family_name][starting_from_version]=...
+# deb.dependency.list=...
+# deb.dependency.list.[os_family_name]=...
+# deb.dependency.list.[os_family_name][starting_from_version]=...
+#
+# Examples:
+# rpm.dependency.list.suse=openssl,\nRequires: rpm-python # for all suse family os
+# rpm.dependency.list.suse11=curl # for all suse family os higher or equal to 11.
+#
+# Such a format is respected by install_ambari_tarball.py by default,
+# however should be encouraged manually in pom.xml.
+
+rpm.dependency.list=postgresql-server >= 8.1,\nRequires: openssl,\nRequires: python >= 2.6
+rpm.dependency.list.suse=postgresql-server >= 8.1,\nRequires: openssl,\nRequires: python-xml,\nRequires: python >= 2.6
+deb.dependency.list=openssl, postgresql (>= 8.1), python (>= 2.6), curl
\ No newline at end of file