You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/04/05 10:04:19 UTC
[buildstream-plugins] 04/49: Initially adding pip source
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/buildstream-plugins.git
commit 8ab4718dd37355fcefc7b9590cbc7e3808cbdd5c
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Fri Mar 18 16:28:53 2022 +0900
Initially adding pip source
From bst-plugins-experimental
---
src/buildstream_plugins/sources/pip.py | 262 +++++++++++++++++++++++++++++++++
1 file changed, 262 insertions(+)
diff --git a/src/buildstream_plugins/sources/pip.py b/src/buildstream_plugins/sources/pip.py
new file mode 100644
index 0000000..21e26a0
--- /dev/null
+++ b/src/buildstream_plugins/sources/pip.py
@@ -0,0 +1,262 @@
+#
+# Copyright 2018 Bloomberg Finance LP
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# Chandan Singh <cs...@bloomberg.net>
+
+"""
+pip - stage python packages using pip
+=====================================
+
+**Host depndencies:**
+
+ * ``pip`` python module
+
+This plugin will download source distributions for specified packages using
+``pip`` but will not install them. It is expected that the elements using this
+source will install the downloaded packages.
+
+Downloaded tarballs will be stored in a directory called ".bst_pip_downloads".
+
+**Usage:**
+
+.. code:: yaml
+
+ # Specify the pip source kind
+ kind: pip
+
+ # Optionally specify index url, defaults to PyPi
+ # This url is used to discover new versions of packages and download them
+ # Projects intending to mirror their sources to a permanent location should
+ # use an aliased url, and declare the alias in the project configuration
+ url: https://mypypi.example.com/simple
+
+ # Optionally specify the path to requirements files
+ # Note that either 'requirements-files' or 'packages' must be defined
+ requirements-files:
+ - requirements.txt
+
+ # Optionally specify a list of additional packages
+ # Note that either 'requirements-files' or 'packages' must be defined
+ packages:
+ - flake8
+
+ # Specify the ref. It is a list of strings of format
+ # "<package-name>==<version>", separated by "\\n".
+ # Usually this will be contents of a requirements.txt file where all
+ # package versions have been frozen.
+ ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"
+
+See `built-in functionality doumentation
+<https://docs.buildstream.build/master/buildstream.source.html#core-source-builtins>`_ for
+details on common configuration options for sources.
+"""
+
+import hashlib
+import os
+import re
+
+from buildstream import Source, SourceError, utils
+
+_OUTPUT_DIRNAME = ".bst_pip_downloads"
+_PYPI_INDEX_URL = "https://pypi.org/simple/"
+
+# Used only for finding pip command
+_PYTHON_VERSIONS = [
+ "python", # when running in a venv, we might not have the exact version
+ "python2.7",
+ "python3.0",
+ "python3.1",
+ "python3.2",
+ "python3.3",
+ "python3.4",
+ "python3.5",
+ "python3.6",
+ "python3.7",
+ "python3.8",
+]
+
+# List of allowed extensions taken from
+# https://docs.python.org/3/distutils/sourcedist.html.
+# Names of source distribution archives must be of the form
+# '%{package-name}-%{version}.%{extension}'.
+_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE,)
+
+
+class PipSource(Source):
+ # pylint: disable=attribute-defined-outside-init
+ BST_MIN_VERSION = "2.0"
+
+ # We need access to previous sources at track time to use requirements.txt
+ # but not at fetch time as self.ref should contain sufficient information
+ # for this plugin
+ BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
+
+ def configure(self, node):
+ node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS)
+ self.ref = node.get_str("ref", None)
+ self.original_url = node.get_str("url", _PYPI_INDEX_URL)
+ self.index_url = self.translate_url(self.original_url)
+ self.packages = node.get_str_list("packages", [])
+ self.requirements_files = node.get_str_list("requirements-files", [])
+
+ if not (self.packages or self.requirements_files):
+ raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self))
+
+ def preflight(self):
+ # Try to find a pip version that spports download command
+ self.host_pip = None
+ for python in reversed(_PYTHON_VERSIONS):
+ try:
+ host_python = utils.get_host_tool(python)
+ rc = self.call([host_python, "-m", "pip", "download", "--help"])
+ if rc == 0:
+ self.host_pip = [host_python, "-m", "pip"]
+ break
+ except utils.ProgramNotFoundError:
+ pass
+
+ if self.host_pip is None:
+ raise SourceError("{}: Unable to find a suitable pip command".format(self))
+
+ def get_unique_key(self):
+ return [self.original_url, self.ref]
+
+ def is_cached(self):
+ return os.path.exists(self._mirror) and os.listdir(self._mirror)
+
+ def get_ref(self):
+ return self.ref
+
+ def load_ref(self, node):
+ self.ref = node.get_str("ref", None)
+
+ def set_ref(self, ref, node):
+ node["ref"] = self.ref = ref
+
+ def track(self, previous_sources_dir): # pylint: disable=arguments-differ
+ # XXX pip does not offer any public API other than the CLI tool so it
+ # is not feasible to correctly parse the requirements file or to check
+ # which package versions pip is going to install.
+ # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
+ # for details.
+ # As a result, we have to wastefully install the packages during track.
+ with self.tempdir() as tmpdir:
+ install_args = self.host_pip + [
+ "download",
+ "--no-binary",
+ ":all:",
+ "--index-url",
+ self.index_url,
+ "--dest",
+ tmpdir,
+ ]
+ for requirement_file in self.requirements_files:
+ fpath = os.path.join(previous_sources_dir, requirement_file)
+ install_args += ["-r", fpath]
+ install_args += self.packages
+
+ self.call(install_args, fail="Failed to install python packages")
+ reqs = self._parse_sdist_names(tmpdir)
+
+ return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
+
+ def fetch(self): # pylint: disable=arguments-differ
+ with self.tempdir() as tmpdir:
+ packages = self.ref.strip().split("\n")
+ package_dir = os.path.join(tmpdir, "packages")
+ os.makedirs(package_dir)
+ self.call(
+ [
+ *self.host_pip,
+ "download",
+ "--no-binary",
+ ":all:",
+ "--index-url",
+ self.index_url,
+ "--dest",
+ package_dir,
+ *packages,
+ ],
+ fail="Failed to install python packages: {}".format(packages),
+ )
+
+ # If the mirror directory already exists, assume that some other
+ # process has fetched the sources before us and ensure that we do
+ # not raise an error in that case.
+ try:
+ utils.move_atomic(package_dir, self._mirror)
+ except utils.DirectoryExistsError:
+ # Another process has beaten us and has fetched the sources
+ # before us.
+ pass
+ except OSError as e:
+ raise SourceError(
+ "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format(
+ self, package_dir, self._mirror, e
+ )
+ ) from e
+
+ def stage(self, directory):
+ with self.timed_activity("Staging Python packages", silent_nested=True):
+ utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME))
+
+ # Directory where this source should stage its files
+ #
+ @property
+ def _mirror(self):
+ if not self.ref:
+ return None
+ return os.path.join(
+ self.get_mirror_directory(),
+ utils.url_directory_name(self.original_url),
+ hashlib.sha256(self.ref.encode()).hexdigest(),
+ )
+
+ # Parse names of downloaded source distributions
+ #
+ # Args:
+ # basedir (str): Directory containing source distribution archives
+ #
+ # Returns:
+ # (list): List of (package_name, version) tuples in sorted order
+ #
+ def _parse_sdist_names(self, basedir):
+ reqs = []
+ for f in os.listdir(basedir):
+ pkg = _match_package_name(f)
+ if pkg is not None:
+ reqs.append(pkg)
+
+ return sorted(reqs)
+
+
+# Extract the package name and version of a source distribution
+#
+# Args:
+# filename (str): Filename of the source distribution
+#
+# Returns:
+# (tuple): A tuple of (package_name, version)
+#
+def _match_package_name(filename):
+ pkg_match = _SDIST_RE.match(filename)
+ if pkg_match is None:
+ return None
+ return pkg_match.groups()
+
+
+def setup():
+ return PipSource