You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@qpid.apache.org by ac...@apache.org on 2015/06/03 19:25:45 UTC
[22/50] [abbrv] qpid-proton git commit: Allow setup.py for bundling
proton
Allow setup.py for bundling proton
As of now, it's not possible to install python-qpid-proton if
libqpid-proton is not present in the system. To be more precises, it's
possible to build python-qpid-proton using cmake, upload it and beg to
the gods of OPs that the required (and correct) shared library will be
present in the system.
This patch adds to python-qpid-proton the ability to download, build and
install qpid-proton if the required version is not present in the
system. It does this by checking - using pkg-config - whether the
required version is installed and if not, it goes to downloading the
package from the official apache source and builds it using cmake.
As nasty as it sounds, this process is not strange in the Python
community. Very famous - and way more used - libraries like PyZMQ (from
which this work took lots of inspiration) do this already in a fairly
more complex way.
This first step is quite simple, it checks, downloads and builds using
the standard tools. It's enabled just for linux and it does not use
fancy flags. Future enhancements could take care of improving the
implementation and extending it to support other systems.
Project: http://git-wip-us.apache.org/repos/asf/qpid-proton/repo
Commit: http://git-wip-us.apache.org/repos/asf/qpid-proton/commit/2e2ba9c7
Tree: http://git-wip-us.apache.org/repos/asf/qpid-proton/tree/2e2ba9c7
Diff: http://git-wip-us.apache.org/repos/asf/qpid-proton/diff/2e2ba9c7
Branch: refs/heads/cjansen-cpp-client
Commit: 2e2ba9c7597293883fc51b231786f7cd49796e28
Parents: d5bd60d
Author: Flavio Percoco <fl...@gmail.com>
Authored: Wed May 13 18:45:07 2015 +0200
Committer: Flavio Percoco <fl...@gmail.com>
Committed: Thu May 14 16:24:45 2015 +0200
----------------------------------------------------------------------
proton-c/bindings/python/PYZMQ_LICENSE.BSD | 32 ++++
proton-c/bindings/python/setup.py | 188 ++++++++++++++++++-
proton-c/bindings/python/setuputils/__init__.py | 0
proton-c/bindings/python/setuputils/bundle.py | 84 +++++++++
proton-c/bindings/python/setuputils/log.py | 46 +++++
proton-c/bindings/python/setuputils/misc.py | 137 ++++++++++++++
6 files changed, 485 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/qpid-proton/blob/2e2ba9c7/proton-c/bindings/python/PYZMQ_LICENSE.BSD
----------------------------------------------------------------------
diff --git a/proton-c/bindings/python/PYZMQ_LICENSE.BSD b/proton-c/bindings/python/PYZMQ_LICENSE.BSD
new file mode 100644
index 0000000..a0a3790
--- /dev/null
+++ b/proton-c/bindings/python/PYZMQ_LICENSE.BSD
@@ -0,0 +1,32 @@
+PyZMQ is licensed under the terms of the Modified BSD License (also known as
+New or Revised BSD), as follows:
+
+Copyright (c) 2009-2012, Brian Granger, Min Ragan-Kelley
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this
+list of conditions and the following disclaimer.
+
+Redistributions in binary form must reproduce the above copyright notice, this
+list of conditions and the following disclaimer in the documentation and/or
+other materials provided with the distribution.
+
+Neither the name of PyZMQ nor the names of its contributors may be used to
+endorse or promote products derived from this software without specific prior
+written permission.
+
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
http://git-wip-us.apache.org/repos/asf/qpid-proton/blob/2e2ba9c7/proton-c/bindings/python/setup.py
----------------------------------------------------------------------
diff --git a/proton-c/bindings/python/setup.py b/proton-c/bindings/python/setup.py
index cdfa104..e61edc7 100755
--- a/proton-c/bindings/python/setup.py
+++ b/proton-c/bindings/python/setup.py
@@ -16,12 +16,195 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-#
+"""
+python-qpid-proton setup script
+
+DISCLAIMER: This script took lots of inspirations from PyZMQ, which is licensed
+under the 'MODIFIED BSD LICENSE'.
+
+Although inspired by the work in PyZMQ, this script and the modules it depends
+on were largely simplified to meet the requirements of the library.
+
+The default behavior of this script is to build the registered `_cproton`
+extension using the system qpid-proton library. However, before doing that, the
+script will try to discover the available qpid-proton's version and whether it's
+suitable for the version of this library. This allows us to have a tight release
+between the versions of the bindings and qpid-proton's version.
+
+The versions used to verify this are in `setuputils.bundle` and they should be
+increased on every release. Note that `bundled_version` matches the current
+released version. The motivation behind this is that we don't know how many
+new releases will be made in the `0.9` series, therefore we need to target the
+latest possible.
+
+If qpid-proton is found in the system and the available versions are match the
+required ones, then the install process will continue normally.
+
+If the available versions are not good for the bindings or the library is
+missing, then the following will happen:
+
+The setup script will attempt to download qpid-proton's tar - see
+`setuputils.bundle.fetch_libqpid_proton` - and it'll build it and then install
+it. Note that if this is being executed outside a virtualenv, it'll install it
+on whatever `distutils.sysconfig.PREFIX` is available. Once qpid-proton has been
+built and installed, the extension build will proceed normally as if the library
+would have been found. The extension will use the recently built library.
+
+While the above removes the need of *always* having qpid-proton installed, it does
+not solve the need of having `cmake` and `swig` installed to make this setup work.
+Eventually, future works should remove the need of `cmake` by collecting sources
+and letting `distutils` do the compilation.
+
+On a final note, it's important to say that in either case, the library paths will
+be added as `rpaths` to the `_cproton` shared library. Mainly because we need to
+make sure `libqpid-proton.so` will be found and loaded when it's not installed in
+the main system.
+
+From the Python side, this scripts overrides 1 command - build_ext - and it adds a
+new one. The later - Configure - is called from the former to setup/discover what's
+in the system. The rest of the comands and steps are done normally without any kind
+of monkey patching.
+"""
+
+import os
+import subprocess
+import sys
+
+import distutils.spawn as ds_spawn
+import distutils.sysconfig as ds_sys
from distutils.core import setup, Extension
+from distutils.command.build import build
+from distutils.command.build_ext import build_ext
+
+from setuputils import bundle
+from setuputils import log
+from setuputils import misc
+
+
+class CustomBuildOrder(build):
+ # NOTE(flaper87): The sole purpose of this class is to re-order
+ # the commands execution so that `build_ext` is executed *before*
+ # build_py. We need this to make sure `cproton.py` is generated
+ # before the python modules are collected. Otherwise, it won't
+ # be installed.
+ sub_commands = [
+ ('build_ext', build.has_ext_modules),
+ ('build_py', build.has_pure_modules),
+ ('build_clib', build.has_c_libraries),
+ ('build_scripts', build.has_scripts),
+ ]
+
+
+class CheckingBuildExt(build_ext):
+ """Subclass build_ext to build qpid-proton using `cmake`"""
+
+ def build_extensions(self):
+ self.check_extensions_list(self.extensions)
+
+ for ext in self.extensions:
+ self.build_extension(ext)
+
+ def build_cmake_proton(self, ext):
+ if not ds_spawn.find_executable('cmake'):
+ log.fatal("cmake needed for this script it work")
+
+ try:
+ ds_spawn.spawn(['cmake'] + ext.extra_compile_args + ext.sources)
+ ds_spawn.spawn(['make', 'install'])
+ except ds_spawn.DistutilsExecError:
+ msg = "Error while running cmake"
+ msg += "\nrun 'setup.py build --help' for build options"
+ log.fatal(msg)
+
+ def build_extension(self, ext):
+ if ext.name == 'cmake_cproton':
+ return self.build_cmake_proton(ext)
+
+ return build_ext.build_extension(self, ext)
+
+ def run(self):
+ # Discover qpid-proton in the system
+ self.distribution.run_command('configure')
+ build_ext.run(self)
+
+
+class Configure(build_ext):
+ description = "Discover Qpid Proton version"
+
+ def bundle_libqpid_proton_extension(self):
+ bundledir = "bundled"
+ ext_modules = self.distribution.ext_modules
+
+ log.info("Using bundled libqpid-proton")
+
+ if not os.path.exists(bundledir):
+ os.makedirs(bundledir)
+
+ bundle.fetch_libqpid_proton(bundledir)
+
+ libqpid_proton_dir = os.path.join(bundledir, 'qpid-proton')
+
+ # NOTE(flaper87): Find prefix. We need to run make ourselves
+ libqpid_proton = Extension(
+ 'cmake_cproton',
+ sources = [libqpid_proton_dir],
+
+ # NOTE(flaper87): Disable all bindings, set the prefix to whatever
+ # is in `distutils.sysconfig.PREFIX` and finally, disable testing
+ # as well. The python binding will be built by this script later
+ # on. Don't let cmake do it.
+ extra_compile_args = ['-DCMAKE_INSTALL_PREFIX:PATH=%s' % ds_sys.PREFIX,
+ '-DBUILD_PYTHON=False',
+ '-DBUILD_JAVA=False',
+ '-DBUILD_PERL=False',
+ '-DBUILD_RUBY=False',
+ '-DBUILD_PHP=False',
+ '-DBUILD_JAVASCRIPT=False',
+ '-DBUILD_TESTING=False',
+ ]
+ )
+
+ # NOTE(flaper87): Register this new extension and make
+ # sure it's built and installed *before* `_cproton`.
+ self.distribution.ext_modules.insert(0, libqpid_proton)
+
+ def set_cproton_settings(self, prefix=None):
+ settings = misc.settings_from_prefix(prefix)
+ ext = self.distribution.ext_modules[-1]
+
+ for k, v in settings.items():
+ setattr(ext, k, v)
+
+ def check_qpid_proton_version(self):
+ """check the qpid_proton version"""
+
+ target_version = bundle.bundled_version_str
+ return (misc.pkg_config_version(max_version=target_version) and
+ misc.pkg_config_version(atleast=bundle.min_qpid_proton_str))
+
+
+ @property
+ def bundle_proton(self):
+ return sys.platform == 'linux2' and not self.check_qpid_proton_version()
+
+ def run(self):
+ prefix = None
+ if self.bundle_proton:
+ self.bundle_libqpid_proton_extension()
+ prefix = ds_sys.PREFIX
+
+ self.set_cproton_settings(prefix)
+
+
+# NOTE(flaper87): Override `build_ext` and add `configure`
+cmdclass = {'configure': Configure,
+ 'build': CustomBuildOrder,
+ 'build_ext': CheckingBuildExt}
+
setup(name='python-qpid-proton',
- version='0.9',
+ version=bundle.min_qpid_proton_str,
description='An AMQP based messaging library.',
author='Apache Qpid',
author_email='proton@qpid.apache.org',
@@ -32,5 +215,6 @@ setup(name='python-qpid-proton',
classifiers=["License :: OSI Approved :: Apache Software License",
"Intended Audience :: Developers",
"Programming Language :: Python"],
+ cmdclass = cmdclass,
ext_modules=[Extension('_cproton', ['cproton.i'],
libraries=['qpid-proton'])])
http://git-wip-us.apache.org/repos/asf/qpid-proton/blob/2e2ba9c7/proton-c/bindings/python/setuputils/__init__.py
----------------------------------------------------------------------
diff --git a/proton-c/bindings/python/setuputils/__init__.py b/proton-c/bindings/python/setuputils/__init__.py
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/qpid-proton/blob/2e2ba9c7/proton-c/bindings/python/setuputils/bundle.py
----------------------------------------------------------------------
diff --git a/proton-c/bindings/python/setuputils/bundle.py b/proton-c/bindings/python/setuputils/bundle.py
new file mode 100644
index 0000000..498e163
--- /dev/null
+++ b/proton-c/bindings/python/setuputils/bundle.py
@@ -0,0 +1,84 @@
+#-----------------------------------------------------------------------------
+# Copyright (C) PyZMQ Developers
+# Distributed under the terms of the Modified BSD License.
+#
+# This bundling code is largely adapted from pyzmq-static's get.sh by
+# Brandon Craig-Rhodes, which is itself BSD licensed.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# This bundling code is largely adapted from pyzmq's code
+# PyZMQ Developers, which is itself Modified BSD licensed.
+#-----------------------------------------------------------------------------
+
+import os
+import shutil
+import stat
+import sys
+import tarfile
+from glob import glob
+from subprocess import Popen, PIPE
+
+try:
+ # py2
+ from urllib2 import urlopen
+except ImportError:
+ # py3
+ from urllib.request import urlopen
+
+from . import log
+
+
+#-----------------------------------------------------------------------------
+# Constants
+#-----------------------------------------------------------------------------
+min_qpid_proton = (0, 9)
+min_qpid_proton_str = "%i.%i" % min_qpid_proton
+
+bundled_version = (0,9,1)
+bundled_version_str = "%i.%i.%i" % (0,9,1)
+libqpid_proton = "qpid-proton-%s.tar.gz" % bundled_version_str
+libqpid_proton_url = ("http://www.apache.org/dist/qpid/proton/%s/%s" %
+ (bundled_version_str, libqpid_proton))
+
+HERE = os.path.dirname(__file__)
+ROOT = os.path.dirname(HERE)
+
+
+def fetch_archive(savedir, url, fname):
+ """Download an archive to a specific location
+
+ :param savedir: Destination dir
+ :param url: URL where the archive should be downloaded from
+ :param fname: Archive's filename
+ """
+ dest = os.path.join(savedir, fname)
+
+ if os.path.exists(dest):
+ log.info("already have %s" % fname)
+ return dest
+
+ log.info("fetching %s into %s" % (url, savedir))
+ if not os.path.exists(savedir):
+ os.makedirs(savedir)
+ req = urlopen(url)
+ with open(dest, 'wb') as f:
+ f.write(req.read())
+ return dest
+
+
+def fetch_libqpid_proton(savedir):
+ """Download qpid-proton to `savedir`."""
+ dest = os.path.join(savedir, 'qpid-proton')
+ if os.path.exists(dest):
+ log.info("already have %s" % dest)
+ return
+ fname = fetch_archive(savedir, libqpid_proton_url, libqpid_proton)
+ tf = tarfile.open(fname)
+ member = tf.firstmember.path
+ if member == '.':
+ member = tf.getmembers()[1].path
+ with_version = os.path.join(savedir, member)
+ tf.extractall(savedir)
+ tf.close()
+ shutil.move(with_version, dest)
http://git-wip-us.apache.org/repos/asf/qpid-proton/blob/2e2ba9c7/proton-c/bindings/python/setuputils/log.py
----------------------------------------------------------------------
diff --git a/proton-c/bindings/python/setuputils/log.py b/proton-c/bindings/python/setuputils/log.py
new file mode 100644
index 0000000..1e051d4
--- /dev/null
+++ b/proton-c/bindings/python/setuputils/log.py
@@ -0,0 +1,46 @@
+#-----------------------------------------------------------------------------
+# Copyright (C) PyZMQ Developers
+# Distributed under the terms of the Modified BSD License.
+#
+# This bundling code is largely adapted from pyzmq-static's get.sh by
+# Brandon Craig-Rhodes, which is itself BSD licensed.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# Logging (adapted from h5py: http://h5py.googlecode.com)
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# This log code is largely adapted from pyzmq's code
+# PyZMQ Developers, which is itself Modified BSD licensed.
+#-----------------------------------------------------------------------------
+
+
+import os
+import sys
+import logging
+
+
+logger = logging.getLogger()
+if os.environ.get('DEBUG'):
+ logger.setLevel(logging.DEBUG)
+else:
+ logger.setLevel(logging.INFO)
+ logger.addHandler(logging.StreamHandler(sys.stderr))
+
+
+def debug(msg):
+ logger.debug(msg)
+
+
+def info(msg):
+ logger.info(msg)
+
+
+def fatal(msg, code=1):
+ logger.error("Fatal: " + msg)
+ exit(code)
+
+
+def warn(msg):
+ logger.error("Warning: " + msg)
http://git-wip-us.apache.org/repos/asf/qpid-proton/blob/2e2ba9c7/proton-c/bindings/python/setuputils/misc.py
----------------------------------------------------------------------
diff --git a/proton-c/bindings/python/setuputils/misc.py b/proton-c/bindings/python/setuputils/misc.py
new file mode 100644
index 0000000..3ebb160
--- /dev/null
+++ b/proton-c/bindings/python/setuputils/misc.py
@@ -0,0 +1,137 @@
+#-----------------------------------------------------------------------------
+# Copyright (C) PyZMQ Developers
+# Distributed under the terms of the Modified BSD License.
+#
+# This bundling code is largely adapted from pyzmq-static's get.sh by
+# Brandon Craig-Rhodes, which is itself BSD licensed.
+#-----------------------------------------------------------------------------
+
+#-----------------------------------------------------------------------------
+# These functions were largely adapted from pyzmq's code
+# PyZMQ Developers, which is itself Modified BSD licensed.
+#-----------------------------------------------------------------------------
+
+
+import os
+import subprocess
+import sys
+
+from . import log
+
+
+def settings_from_prefix(prefix=None):
+ """Load appropriate library/include settings from qpid-proton prefix
+
+ NOTE: Funcion adapted from PyZMQ, which is itself Modified BSD licensed.
+
+ :param prefix: The install prefix where the dependencies are expected to be
+ """
+ settings = {}
+ settings['libraries'] = []
+ settings['include_dirs'] = []
+ settings['library_dirs'] = []
+ settings['swig_opts'] = []
+ settings['runtime_library_dirs'] = []
+ settings['extra_link_args'] = []
+
+ if sys.platform.startswith('win'):
+ settings['libraries'].append('libqpid-proton')
+
+ if prefix:
+ settings['include_dirs'] += [os.path.join(prefix, 'include')]
+ settings['library_dirs'] += [os.path.join(prefix, 'lib')]
+ else:
+
+ # If prefix is not explicitly set, pull it from pkg-config by default.
+
+ if not prefix:
+ try:
+ cmd = 'pkg-config --variable=prefix --print-errors libqpid-proton'.split()
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ log.info("pkg-config not found")
+ else:
+ log.warn("Running pkg-config failed - %s." % e)
+ p = None
+ if p is not None:
+ if p.wait():
+ log.info("Did not find libqpid-proton via pkg-config:")
+ log.info(p.stderr.read().decode())
+ else:
+ prefix = p.stdout.readline().strip().decode()
+ log.info("Using qpid-proton-prefix %s (found via pkg-config)." % prefix)
+
+ settings['libraries'].append('qpid-proton')
+ # add pthread on freebsd
+ if sys.platform.startswith('freebsd'):
+ settings['libraries'].append('pthread')
+
+ if sys.platform == 'sunos5':
+ if platform.architecture()[0] == '32bit':
+ settings['extra_link_args'] += ['-m32']
+ else:
+ settings['extra_link_args'] += ['-m64']
+ if prefix:
+ settings['include_dirs'] += [os.path.join(prefix, 'include')]
+ settings['library_dirs'] += [os.path.join(prefix, 'lib')]
+ settings['library_dirs'] += [os.path.join(prefix, 'lib64')]
+
+ else:
+ if sys.platform == 'darwin' and os.path.isdir('/opt/local/lib'):
+ # allow macports default
+ settings['include_dirs'] += ['/opt/local/include']
+ settings['library_dirs'] += ['/opt/local/lib']
+ settings['library_dirs'] += ['/opt/local/lib64']
+ if os.environ.get('VIRTUAL_ENV', None):
+ # find libqpid_proton installed in virtualenv
+ env = os.environ['VIRTUAL_ENV']
+ settings['include_dirs'] += [os.path.join(env, 'include')]
+ settings['library_dirs'] += [os.path.join(env, 'lib')]
+ settings['library_dirs'] += [os.path.join(env, 'lib64')]
+
+ if sys.platform != 'darwin':
+ settings['runtime_library_dirs'] += [
+ os.path.abspath(x) for x in settings['library_dirs']
+ ]
+
+ for d in settings['include_dirs']:
+ settings['swig_opts'] += ['-I' + d]
+
+ return settings
+
+
+def pkg_config_version(atleast=None, max_version=None):
+ """Check the qpid_proton version using pkg-config
+
+ This function returns True/False depending on whether
+ the library is found and atleast/max_version are met.
+
+ :param atleast: The minimum required version
+ :param max_version: The maximum supported version. This
+ basically represents the target version.
+ """
+
+ if atleast and max_version:
+ log.error('Specify either atleast or max_version')
+
+ try:
+ cmd = ['pkg-config',
+ '--%s-version=%s' % (atleast and 'atleast' or 'max',
+ atleast or max_version),
+ 'libqpid-proton']
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ log.info("pkg-config not found")
+ else:
+ log.warn("Running pkg-config failed - %s." % e)
+ return False
+
+ if p.wait():
+ log.info("Did not find libqpid-proton via pkg-config:")
+ log.info(p.stderr.read().decode())
+ return False
+
+ log.info("Using libqpid-proton (found via pkg-config).")
+ return True
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@qpid.apache.org
For additional commands, e-mail: commits-help@qpid.apache.org