You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/04/05 10:04:48 UTC
[buildstream-plugins] 33/49: tests/elements/pip.py: Adding tests for pip element
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/buildstream-plugins.git
commit 8b030e9eb33cad067011db90a3da91fab35518bd
Author: Tristan van Berkom <tr...@codethink.co.uk>
AuthorDate: Mon Mar 21 16:54:35 2022 +0900
tests/elements/pip.py: Adding tests for pip element
---
tests/elements/pip.py | 132 ++++++++++++++++++++++
tests/elements/pip/elements/base.bst | 3 +
tests/elements/pip/elements/base/alpine-image.bst | 6 +
tests/elements/pip/files/piphello.tar.xz | Bin 0 -> 628 bytes
tests/elements/pip/project.conf | 15 +++
tests/testutils/python_repo.py | 132 ++++++++++++++++++++++
6 files changed, 288 insertions(+)
diff --git a/tests/elements/pip.py b/tests/elements/pip.py
new file mode 100644
index 0000000..1664063
--- /dev/null
+++ b/tests/elements/pip.py
@@ -0,0 +1,132 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+
+import pytest
+
+from buildstream import _yaml
+
+from buildstream._testing import cli_integration as cli # pylint: disable=unused-import
+from buildstream._testing.integration import assert_contains
+from buildstream._testing.integration import integration_cache # pylint: disable=unused-import
+from buildstream._testing._utils.site import HAVE_SANDBOX
+
+from tests.testutils.python_repo import setup_pypi_repo # pylint: disable=unused-import
+
+
+pytestmark = pytest.mark.integration
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_pip_build(cli, datafiles):
+ project = str(datafiles)
+ checkout = os.path.join(cli.directory, "checkout")
+ element_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
+
+ element = {
+ "kind": "pip",
+ "variables": {"pip": "pip3"},
+ "depends": [{"filename": "base.bst"}],
+ "sources": [
+ {
+ "kind": "tar",
+ "url": "file://{}/files/piphello.tar.xz".format(project),
+ "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d",
+ }
+ ],
+ }
+ os.makedirs(
+ os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True,
+ )
+ _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
+
+ result = cli.run(project=project, args=["build", element_name])
+ assert result.exit_code == 0
+
+ result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout],)
+ assert result.exit_code == 0
+
+ assert_contains(
+ checkout, ["/usr", "/usr/lib", "/usr/bin", "/usr/bin/hello", "/usr/lib/python3.6",],
+ )
+
+
+# Test running an executable built with pip
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_pip_run(cli, datafiles):
+ # Create and build our test element
+ test_pip_build(cli, datafiles)
+
+ project = str(datafiles)
+ element_name = "pip/hello.bst"
+
+ result = cli.run(project=project, args=["shell", element_name, "/usr/bin/hello"])
+ assert result.exit_code == 0
+ assert result.output == "Hello, world!\n"
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
+def test_pip_element_should_install_pip_deps(cli, datafiles, setup_pypi_repo):
+ project = str(datafiles)
+ elements_path = os.path.join(project, "elements")
+ element_name = "pip/hello.bst"
+
+ # check that exotically named packages are imported correctly
+ myreqs_packages = "alohalib"
+ dependencies = [
+ "app2",
+ "app.3",
+ "app-4",
+ "app_5",
+ "app.no.6",
+ "app-no-7",
+ "app_no_8",
+ ]
+ mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
+
+ # set up directories
+ pypi_repo = os.path.join(project, "files", "pypi-repo")
+ os.makedirs(pypi_repo, exist_ok=True)
+ os.makedirs(
+ os.path.dirname(os.path.join(elements_path, element_name)), exist_ok=True,
+ )
+ setup_pypi_repo(mock_packages, pypi_repo)
+
+ # create pip element
+ element = {
+ "kind": "pip",
+ "variables": {"pip": "pip3"},
+ "depends": [{"filename": "base.bst"}],
+ "sources": [
+ {
+ "kind": "tar",
+ "url": "file://{}/files/piphello.tar.xz".format(project),
+ # FIXME: remove hardcoded ref once issue #1010 is closed
+ "ref": "ad96570b552498807abec33c06210bf68378d854ced6753b77916c5ed517610d",
+ },
+ {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages],},
+ ],
+ }
+ _yaml.roundtrip_dump(element, os.path.join(elements_path, element_name))
+
+ result = cli.run(project=project, args=["source", "track", element_name])
+ assert result.exit_code == 0
+
+ result = cli.run(project=project, args=["build", element_name])
+ assert result.exit_code == 0
+
+ # get installed packages in sandbox
+ installed_packages = set(
+ cli.run(project=project, args=["shell", element_name, "pip3", "freeze"]).output.split("\n")
+ )
+ # compare with packages that are expected to be installed
+ pip_source_packages = {package.replace("_", "-") + "==0.1" for package in dependencies + [myreqs_packages]}
+ assert pip_source_packages.issubset(installed_packages)
diff --git a/tests/elements/pip/elements/base.bst b/tests/elements/pip/elements/base.bst
new file mode 100644
index 0000000..da7c70b
--- /dev/null
+++ b/tests/elements/pip/elements/base.bst
@@ -0,0 +1,3 @@
+kind: stack
+depends:
+- base/alpine-image.bst
diff --git a/tests/elements/pip/elements/base/alpine-image.bst b/tests/elements/pip/elements/base/alpine-image.bst
new file mode 100644
index 0000000..f8e00ba
--- /dev/null
+++ b/tests/elements/pip/elements/base/alpine-image.bst
@@ -0,0 +1,6 @@
+kind: import
+description: Import an alpine image as the platform
+sources:
+- kind: tar
+ url: alpine:integration-tests-base.v1.x86_64.tar.xz
+ ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639
diff --git a/tests/elements/pip/files/piphello.tar.xz b/tests/elements/pip/files/piphello.tar.xz
new file mode 100644
index 0000000..72ec9b3
Binary files /dev/null and b/tests/elements/pip/files/piphello.tar.xz differ
diff --git a/tests/elements/pip/project.conf b/tests/elements/pip/project.conf
new file mode 100644
index 0000000..f04cd98
--- /dev/null
+++ b/tests/elements/pip/project.conf
@@ -0,0 +1,15 @@
+# test project config
+name: test
+min-version: 2.0
+
+element-path: elements
+
+plugins:
+- origin: pip
+ package-name: buildstream-plugins
+ elements:
+ - pip
+
+aliases:
+ alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/
+ project_dir: file://{project_dir}
diff --git a/tests/testutils/python_repo.py b/tests/testutils/python_repo.py
new file mode 100644
index 0000000..52dd8a6
--- /dev/null
+++ b/tests/testutils/python_repo.py
@@ -0,0 +1,132 @@
+import os
+import re
+import shutil
+import subprocess
+import sys
+
+import pytest
+
+
+SETUP_TEMPLATE = """\
+from setuptools import setup
+
+setup(
+ name='{name}',
+ version='{version}',
+ description='{name}',
+ packages=['{pkgdirname}'],
+ install_requires={pkgdeps},
+ entry_points={{
+ 'console_scripts': [
+ '{pkgdirname}={pkgdirname}:main'
+ ]
+ }}
+)
+"""
+
+# All packages generated via generate_pip_package will have the functions below
+INIT_TEMPLATE = """\
+def main():
+ print('This is {name}')
+
+def hello(actor='world'):
+ print('Hello {{}}! This is {name}'.format(actor))
+"""
+
+HTML_TEMPLATE = """\
+<html>
+ <head>
+ <title>Links for {name}</title>
+ </head>
+ <body>
+ <a href='{name}-{version}.tar.gz'>{name}-{version}.tar.gz</a><br />
+ </body>
+</html>
+"""
+
+
+# Creates a simple python source distribution and copies this into a specified
+# directory which is to serve as a mock python repository
+#
+# Args:
+# tmpdir (str): Directory in which the source files will be created
+# pypi (str): Directory serving as a mock python repository
+# name (str): The name of the package to be created
+# version (str): The version of the package to be created
+#
+# Returns:
+# None
+#
+def generate_pip_package(tmpdir, pypi, name, version="0.1", dependencies=None):
+ if dependencies is None:
+ dependencies = []
+ # check if package already exists in pypi
+ pypi_package = os.path.join(pypi, re.sub("[^0-9a-zA-Z]+", "-", name))
+ if os.path.exists(pypi_package):
+ return
+
+ # create the package source files in tmpdir resulting in a directory
+ # tree resembling the following structure:
+ #
+ # tmpdir
+ # |-- setup.py
+ # `-- package
+ # `-- __init__.py
+ #
+ setup_file = os.path.join(tmpdir, "setup.py")
+ pkgdirname = re.sub("[^0-9a-zA-Z]+", "", name)
+ with open(setup_file, "w", encoding="utf-8") as f:
+ f.write(SETUP_TEMPLATE.format(name=name, version=version, pkgdirname=pkgdirname, pkgdeps=dependencies,))
+ os.chmod(setup_file, 0o755)
+
+ package = os.path.join(tmpdir, pkgdirname)
+ os.makedirs(package)
+
+ main_file = os.path.join(package, "__init__.py")
+ with open(main_file, "w", encoding="utf-8") as f:
+ f.write(INIT_TEMPLATE.format(name=name))
+ os.chmod(main_file, 0o644)
+
+ # Run sdist with a fresh process
+ subprocess.run([sys.executable, "setup.py", "sdist"], cwd=tmpdir, check=True)
+
+ # create directory for this package in pypi resulting in a directory
+ # tree resembling the following structure:
+ #
+ # pypi
+ # `-- pypi_package
+ # |-- index.html
+ # `-- foo-0.1.tar.gz
+ #
+ os.makedirs(pypi_package)
+
+ # add an index html page
+ index_html = os.path.join(pypi_package, "index.html")
+ with open(index_html, "w", encoding="utf-8") as f:
+ f.write(HTML_TEMPLATE.format(name=name, version=version))
+
+ # copy generated tarfile to pypi package
+ dist_dir = os.path.join(tmpdir, "dist")
+ for tar in os.listdir(dist_dir):
+ tarpath = os.path.join(dist_dir, tar)
+ shutil.copy(tarpath, pypi_package)
+
+
+@pytest.fixture
+def setup_pypi_repo(tmpdir):
+ def create_pkgdir(package):
+ pkgdirname = re.sub("[^0-9a-zA-Z]+", "", package)
+ pkgdir = os.path.join(str(tmpdir), pkgdirname)
+ os.makedirs(pkgdir)
+ return pkgdir
+
+ def add_packages(packages, pypi_repo):
+ for package, dependencies in packages.items():
+ pkgdir = create_pkgdir(package)
+ generate_pip_package(
+ pkgdir, pypi_repo, package, dependencies=list(dependencies.keys()),
+ )
+ for dependency, dependency_dependencies in dependencies.items():
+ add_packages({dependency: dependency_dependencies}, pypi_repo)
+
+ return add_packages