You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@buildstream.apache.org by tv...@apache.org on 2022/08/17 08:08:27 UTC
[buildstream] 01/06: Add option to bundle BuildBox binaries inside Python package
This is an automated email from the ASF dual-hosted git repository.
tvb pushed a commit to branch tristan/build-wheels
in repository https://gitbox.apache.org/repos/asf/buildstream.git
commit 39b3916938d39626b33520b95f7305d9e77ba985
Author: Sam Thursfield <sa...@codethink.co.uk>
AuthorDate: Wed Aug 3 14:01:44 2022 +0200
Add option to bundle BuildBox binaries inside Python package
BuildBox is not widely distributed in binary form yet. For convience,
add a mechanism to bundle prebuilt binaries in the Python wheel
packages.
Setting BST_BUNDLE_BUILDBOX=1 when setup.py runs, causes the bundled
binaries to be included in the binary package. Its up to the packager
to make appropriate binaries available in
`src/buildstream/subprojects/buildbox/`.
BuildStream will search the package subprojects/ dir when looking for
BuildBox binaries on the host in all cases, prioritizing any bundled
binaries above other ones on the host.
Related to #1712
---
MANIFEST.in | 3 ++
pyproject.toml | 23 +++++++++++
setup.py | 57 +++++++++++++++++++++++++-
src/buildstream/_cas/casdprocessmanager.py | 7 +++-
src/buildstream/_site.py | 3 ++
src/buildstream/_testing/_utils/site.py | 4 +-
src/buildstream/sandbox/_sandboxbuildboxrun.py | 8 +++-
src/buildstream/utils.py | 42 +++++++++++++++----
tests/integration/cachedfail.py | 2 +-
tests/sandboxes/selection.py | 2 +-
10 files changed, 134 insertions(+), 17 deletions(-)
diff --git a/MANIFEST.in b/MANIFEST.in
index 71d24737c..9c28ca355 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -47,3 +47,6 @@ include versioneer.py
# setuptools.build_meta don't include setup.py by default. Add it
include setup.py
+
+# bundled binaries should only be in the bdist packages
+recursive-exclude src/buildstream/subprojects *
diff --git a/pyproject.toml b/pyproject.toml
index fefbbecd4..f3eadb7af 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -24,3 +24,26 @@ exclude = '''
| src/buildstream/_protos
)
'''
+
+[tool.cibuildwheel]
+build-frontend = "build"
+environment = { BST_BUNDLE_BUILDBOX = "1" }
+
+# The BuildBox binaries produced in buildbox-integration are linked against GLIBC 2.28
+# from Debian 10. See: https://gitlab.com/BuildGrid/buildbox/buildbox-integration.
+#
+# The PyPA manylinux_2_28 platform tag identifies that our wheel will run on any x86_64
+# OS with GLIBC >= 2.28. Following this setting, `cibuildwheel` builds the packages in
+# the corresponding manylinux_2_28 container image. See: https://github.com/pypa/manylinux
+manylinux-x86_64-image = "manylinux_2_28"
+
+skip = [
+ # BuildStream supports Python >= 3.7
+ "cp36-*",
+ # PyPy may work, but nobody is testing it so avoid distributing prebuilt binaries.
+ "pp*",
+ # Skipping this niche archicture ~halves overall build time.
+ "*_i686",
+ # The prebuilt BuildBox binaries link against GLibc so will work on manylinux but not musllinux
+ "*-musllinux_*",
+]
diff --git a/setup.py b/setup.py
index ade004938..820b0588c 100755
--- a/setup.py
+++ b/setup.py
@@ -64,6 +64,60 @@ except ImportError:
sys.exit(1)
+############################################################
+# List the BuildBox binaries to ship in the wheel packages #
+############################################################
+#
+# BuildBox isn't widely available in OS distributions. To enable a "one click"
+# install for BuildStream, we bundle prebuilt BuildBox binaries in our binary
+# wheel packages.
+#
+# The binaries are provided by the buildbox-integration Gitlab project:
+# https://gitlab.com/BuildGrid/buildbox/buildbox-integration
+#
+# If you want to build a wheel with the BuildBox binaries included, set the
+# env var "BST_BUNDLE_BUILDBOX=1" when running setup.py.
+
+try:
+ BUNDLE_BUILDBOX = int(os.environ.get("BST_BUNDLE_BUILDBOX", "0"))
+except ValueError:
+ print("BST_BUNDLE_BUILDBOX must be an integer. Please set it to '1' to enable, '0' to disable", file=sys.stderr)
+ raise SystemExit(1)
+
+
+def list_buildbox_binaries():
+ expected_binaries = [
+ "buildbox-casd",
+ "buildbox-fuse",
+ "buildbox-run",
+ ]
+
+ if BUNDLE_BUILDBOX:
+ bst_package_dir = Path(__file__).parent.joinpath("src/buildstream")
+ buildbox_dir = bst_package_dir.joinpath("subprojects", "buildbox")
+ buildbox_binaries = [buildbox_dir.joinpath(name) for name in expected_binaries]
+
+ missing_binaries = [path for path in buildbox_binaries if not path.is_file()]
+ if missing_binaries:
+ paths_text = "\n".join([" * {}".format(path) for path in missing_binaries])
+ print(
+ "Expected BuildBox binaries were not found. "
+ "Set BST_BUNDLE_BUILDBOX=0 or provide:\n\n"
+ "{}\n".format(paths_text),
+ file=sys.stderr,
+ )
+ raise SystemExit(1)
+
+ for path in buildbox_binaries:
+ if path.is_symlink():
+ print("Bundled BuildBox binaries must not be symlinks. Please fix {}".format(path))
+ raise SystemExit(1)
+
+ return [str(path.relative_to(bst_package_dir)) for path in buildbox_binaries]
+ else:
+ return []
+
+
###########################################
# List the pre-built man pages to install #
###########################################
@@ -351,7 +405,7 @@ setup(
},
python_requires="~={}.{}".format(REQUIRED_PYTHON_MAJOR, REQUIRED_PYTHON_MINOR),
package_dir={"": "src"},
- packages=find_packages(where="src", exclude=("tests", "tests.*")),
+ packages=find_packages(where="src", exclude=("subprojects", "tests", "tests.*")),
package_data={
"buildstream": [
"py.typed",
@@ -359,6 +413,7 @@ setup(
"plugins/*/*.yaml",
"data/*.yaml",
"data/*.sh.in",
+ *list_buildbox_binaries(),
*list_testing_datafiles(),
]
},
diff --git a/src/buildstream/_cas/casdprocessmanager.py b/src/buildstream/_cas/casdprocessmanager.py
index e7a73c1b2..2b702b925 100644
--- a/src/buildstream/_cas/casdprocessmanager.py
+++ b/src/buildstream/_cas/casdprocessmanager.py
@@ -73,7 +73,7 @@ class CASDProcessManager:
# Early version check
self._check_casd_version(messenger)
- casd_args = [utils.get_host_tool("buildbox-casd")]
+ casd_args = [self.__buildbox_casd()]
casd_args.append("--bind=" + self._connection_string)
casd_args.append("--log-level=" + log_level.value)
@@ -106,6 +106,9 @@ class CASDProcessManager:
casd_args, cwd=path, stdout=logfile_fp, stderr=subprocess.STDOUT, preexec_fn=os.setpgrp
)
+ def __buildbox_casd(self):
+ return utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox")
+
# _check_casd_version()
#
# Check for minimal acceptable version of buildbox-casd.
@@ -121,7 +124,7 @@ class CASDProcessManager:
# We specify a trailing "path" argument because some versions of buildbox-casd
# require specifying the storage path even for invoking the --version option.
#
- casd_args = [utils.get_host_tool("buildbox-casd")]
+ casd_args = [self.__buildbox_casd()]
casd_args.append("--version")
casd_args.append("/")
diff --git a/src/buildstream/_site.py b/src/buildstream/_site.py
index acc7fae1b..62dd7b583 100644
--- a/src/buildstream/_site.py
+++ b/src/buildstream/_site.py
@@ -43,3 +43,6 @@ build_all_template = os.path.join(root, "data", "build-all.sh.in")
# Module building script template
build_module_template = os.path.join(root, "data", "build-module.sh.in")
+
+# The bundled subprojects directory
+subprojects = os.path.join(root, "subprojects")
diff --git a/src/buildstream/_testing/_utils/site.py b/src/buildstream/_testing/_utils/site.py
index 3ce922365..d0e4cdbe0 100644
--- a/src/buildstream/_testing/_utils/site.py
+++ b/src/buildstream/_testing/_utils/site.py
@@ -55,7 +55,7 @@ try:
except ProgramNotFoundError:
HAVE_LZIP = False
-casd_path = utils.get_host_tool("buildbox-casd")
+casd_path = utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox")
CASD_SEPARATE_USER = bool(os.stat(casd_path).st_mode & stat.S_ISUID)
del casd_path
@@ -68,7 +68,7 @@ HAVE_SANDBOX = None
BUILDBOX_RUN = None
try:
- path = utils.get_host_tool("buildbox-run")
+ path = utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox")
subprocess.run([path, "--capabilities"], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
BUILDBOX_RUN = os.path.basename(os.readlink(path))
HAVE_SANDBOX = "buildbox-run"
diff --git a/src/buildstream/sandbox/_sandboxbuildboxrun.py b/src/buildstream/sandbox/_sandboxbuildboxrun.py
index 765caf166..058eedaaa 100644
--- a/src/buildstream/sandbox/_sandboxbuildboxrun.py
+++ b/src/buildstream/sandbox/_sandboxbuildboxrun.py
@@ -34,10 +34,14 @@ from ._sandboxreapi import SandboxREAPI
# BuildBox-based sandbox implementation.
#
class SandboxBuildBoxRun(SandboxREAPI):
+ @classmethod
+ def __buildbox_run(cls):
+ return utils._get_host_tool_internal("buildbox-run", search_subprojects_dir="buildbox")
+
@classmethod
def check_available(cls):
try:
- path = utils.get_host_tool("buildbox-run")
+ path = cls.__buildbox_run()
except utils.ProgramNotFoundError as Error:
cls._dummy_reasons += ["buildbox-run not found"]
raise SandboxError(" and ".join(cls._dummy_reasons), reason="unavailable-local-sandbox") from Error
@@ -92,7 +96,7 @@ class SandboxBuildBoxRun(SandboxREAPI):
action_file.flush()
buildbox_command = [
- utils.get_host_tool("buildbox-run"),
+ self.__buildbox_run(),
"--remote={}".format(casd_process_manager._connection_string),
"--action={}".format(action_file.name),
"--action-result={}".format(result_file.name),
diff --git a/src/buildstream/utils.py b/src/buildstream/utils.py
index 0ea155d58..26f7925d3 100644
--- a/src/buildstream/utils.py
+++ b/src/buildstream/utils.py
@@ -54,6 +54,7 @@ from . import _signals
from ._exceptions import BstError
from .exceptions import ErrorDomain
from ._protos.build.bazel.remote.execution.v2 import remote_execution_pb2
+from . import _site
# Contains utils that have been rewritten in Cython for speed benefits
# This makes them available when importing from utils
@@ -575,7 +576,9 @@ def link_files(
return result
-def get_host_tool(name: str) -> str:
+def get_host_tool(
+ name: str,
+) -> str:
"""Get the full path of a host tool
Args:
@@ -587,13 +590,7 @@ def get_host_tool(name: str) -> str:
Raises:
:class:`.ProgramNotFoundError`
"""
- search_path = os.environ.get("PATH")
- program_path = shutil.which(name, path=search_path)
-
- if not program_path:
- raise ProgramNotFoundError("Did not find '{}' in PATH: {}".format(name, search_path))
-
- return program_path
+ return _get_host_tool_internal(name)
def get_bst_version() -> Tuple[int, int]:
@@ -749,6 +746,35 @@ def get_umask():
return _UMASK
+# _get_host_tool_internal():
+#
+# Get the full path of a host tool, including tools bundled inside the Python package.
+#
+# Args:
+# name (str): The name of the program to search for
+# search_subprojects_dir (str): Optionally search in bundled subprojects directory
+#
+# Returns:
+# The full path to the program, if found
+#
+# Raises:
+# :class:`.ProgramNotFoundError`
+def _get_host_tool_internal(
+ name: str,
+ search_subprojects_dir: Optional[str] = None,
+) -> str:
+ search_path = os.environ.get("PATH", "").split(os.pathsep)
+ if search_subprojects_dir:
+ search_path.insert(0, os.path.join(_site.subprojects, search_subprojects_dir))
+
+ program_path = shutil.which(name, path=os.pathsep.join(search_path))
+
+ if not program_path:
+ raise ProgramNotFoundError("Did not find '{}' in PATH: {}".format(name, search_path))
+
+ return program_path
+
+
# _get_dir_size():
#
# Get the disk usage of a given directory in bytes.
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index da1fff6c2..2700f6e11 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -223,7 +223,7 @@ def test_host_tools_errors_are_not_cached(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
buildbox_casd = tmp_path.joinpath("bin/buildbox-casd")
buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd))
+ os.symlink(utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox"), str(buildbox_casd))
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")
diff --git a/tests/sandboxes/selection.py b/tests/sandboxes/selection.py
index 403ff1f88..efb22dcc4 100644
--- a/tests/sandboxes/selection.py
+++ b/tests/sandboxes/selection.py
@@ -33,7 +33,7 @@ def test_dummy_sandbox_fallback(cli, datafiles, tmp_path):
# Create symlink to buildbox-casd to work with custom PATH
buildbox_casd = tmp_path.joinpath("bin/buildbox-casd")
buildbox_casd.parent.mkdir()
- os.symlink(utils.get_host_tool("buildbox-casd"), str(buildbox_casd))
+ os.symlink(utils._get_host_tool_internal("buildbox-casd", search_subprojects_dir="buildbox"), str(buildbox_casd))
project = str(datafiles)
element_path = os.path.join(project, "elements", "element.bst")